Initial import of Buildbot 0.7.10p1 from release tarball.
authorBen Hearsum <bhearsum@mozilla.com>
Tue, 07 Apr 2009 09:56:39 -0400
changeset 14 54f7d8c5c4e8278a1294eb9b332e197a57da881f
parent 0 9e5c462732f9cfe3c0228845e81065c3c2d94e00
child 15 bf2b04e757b4c8b0ce4ee210a2455065ebd77816
child 22 6515c595f2aa98d10f945b2d024ecaa9f9965488
push id5
push userbhearsum@mozilla.com
push dateWed, 29 Apr 2009 14:10:28 +0000
Initial import of Buildbot 0.7.10p1 from release tarball.
CREDITS
NEWS
PKG-INFO
buildbot.egg-info/PKG-INFO
buildbot.egg-info/SOURCES.txt
buildbot/__init__.py
buildbot/buildslave.py
buildbot/changes/changes.py
buildbot/changes/hgbuildbot.py
buildbot/changes/pb.py
buildbot/changes/svnpoller.py
buildbot/clients/gtkPanes.py
buildbot/clients/sendchange.py
buildbot/interfaces.py
buildbot/master.py
buildbot/process/base.py
buildbot/process/builder.py
buildbot/process/buildstep.py
buildbot/process/factory.py
buildbot/process/properties.py
buildbot/process/step_twisted2.py
buildbot/scheduler.py
buildbot/scripts/checkconfig.py
buildbot/scripts/logwatcher.py
buildbot/scripts/runner.py
buildbot/scripts/sample.cfg
buildbot/scripts/tryclient.py
buildbot/slave/bot.py
buildbot/slave/commands.py
buildbot/status/base.py
buildbot/status/builder.py
buildbot/status/client.py
buildbot/status/mail.py
buildbot/status/web/base.py
buildbot/status/web/baseweb.py
buildbot/status/web/build.py
buildbot/status/web/builder.py
buildbot/status/web/changes.py
buildbot/status/web/classic.css
buildbot/status/web/grid.py
buildbot/status/web/logs.py
buildbot/status/web/slaves.py
buildbot/status/web/waterfall.py
buildbot/status/web/xmlrpc.py
buildbot/status/words.py
buildbot/steps/dummy.py
buildbot/steps/maxq.py
buildbot/steps/python.py
buildbot/steps/python_twisted.py
buildbot/steps/shell.py
buildbot/steps/source.py
buildbot/steps/transfer.py
buildbot/steps/trigger.py
buildbot/test/runutils.py
buildbot/test/test_buildreq.py
buildbot/test/test_buildstep.py
buildbot/test/test_changes.py
buildbot/test/test_config.py
buildbot/test/test_control.py
buildbot/test/test_locks.py
buildbot/test/test_mailparse.py
buildbot/test/test_properties.py
buildbot/test/test_run.py
buildbot/test/test_scheduler.py
buildbot/test/test_slaves.py
buildbot/test/test_status.py
buildbot/test/test_steps.py
buildbot/test/test_transfer.py
buildbot/test/test_vc.py
buildbot/test/test_web.py
buildbot/util.py
contrib/README.txt
contrib/arch_buildbot.py
contrib/bb_applet.py
contrib/darcs_buildbot.py
contrib/fakechange.py
contrib/git_buildbot.py
contrib/hg_buildbot.py
contrib/run_maxq.py
contrib/svn_buildbot.py
contrib/svn_watcher.py
contrib/svnpoller.py
contrib/viewcvspoll.py
contrib/windows/buildbot.bat
contrib/windows/buildbot_service.py
contrib/windows/setup.py
docs/buildbot.html
docs/buildbot.info
docs/buildbot.info-1
docs/buildbot.info-2
docs/buildbot.texinfo
docs/images/master.png
docs/images/overview.png
docs/images/slavebuilder.png
docs/images/slaves.png
docs/images/status.png
setup.py
--- a/CREDITS
+++ b/CREDITS
@@ -3,72 +3,81 @@ no particular order. Thanks everybody!
 
 Aaron Hsieh
 Albert Hofkamp
 Alexander Lorenz
 Alexander Staubo
 AllMyData.com
 Andrew Bennetts
 Anthony Baxter
+Axel Hecht
 Baptiste Lepilleur
 Bear
 Ben Hearsum
 Benoit Sigoure
 Brad Hards
 Brandon Philips
 Brett Neely
 Charles Lepple
+Chad Metcalf
 Christian Unger
 Clement Stenac
 Dan Locks
 Dave Liebreich
 Dave Peticolas
 Dobes Vandermeer
 Dustin Mitchell
+Dustin Sallings
 Elliot Murphy
 Fabrice Crestois
 Gary Granger
+Gary Poster
 Gerald Combs
 Greg Ward
 Grig Gheorghiu
 Haavard Skinnemoen
+Igor Slepchin
 JP Calderone
 James Knight
 Jerome Davann
 John Backstrand
 John O'Duinn
 John Pye
 John Saxton
 Jose Dapena Paz
 Kevin Turner
 Kirill Lapshin
+Marcus Lindblom
 Marius Gedminas
 Mark Dillavou
 Mark Hammond
 Mark Pauley
 Mark Rowe
 Mateusz Loskot
 Nathaniel Smith
 Neal Norwitz
 Nick Mathewson
 Nick Trout
 Niklaus Giger
+Neil Hemingway
 Olivier Bonnet
 Olly Betts
 Paul Warren
 Paul Winkler
 Phil Thompson
 Philipp Frauenfelder
 Rene Rivera
 Riccardo Magliocchetti
 Rob Helmer
 Roch Gadsdon
 Roy Rapoport
 Scott Lamb
 Stephen Davis
+Steve 'Ashcrow' Milner
 Steven Walter
 Ted Mielczarek
 Thomas Vander Stichele
 Tobi Vollebregt
 Wade Brainerd
 Yoz Grahame
 Zandr Milewski
 chops
+zooko
--- a/NEWS
+++ b/NEWS
@@ -1,10 +1,195 @@
 User visible changes in Buildbot.             -*- outline -*-
 
+* Release 0.7.10p1 (2 Mar 2009)
+
+This is a bugfix release for 0.7.10, fixing a few minor bugs:
+
+** Bugs Fixed
+
+*** add a missing method to the IRC status plugin
+
+*** add RPM-related buildsteps to setup.py
+
+* Release 0.7.10 (25 Feb 2009)
+
+This release is mainly a collection of user-submitted patches since
+the last release.
+
+** New Features
+
+*** Environment variables in a builder (#100)
+
+It is useful to be able to pass environment variables to all steps in a
+builder.  This is now possible by adding { .. 'env': { 'var' : 'value' }, ... }
+to the builder specification.
+
+*** IRC status plugin improvements (#330, #357, #378, #280, #381, #411, #368)
+
+*** usePTY specified in master.cfg, defaults to False (#158, #255)
+
+Using a pty has some benefits in terms of supporting "Stop Build", but causes
+numerous problems with simpler jobs which can be killed by a SIGHUP when their
+standard input is closed.  With this change, PTYs are not used by default,
+although you can enable them either on slaves (with the --usepty option to
+create-slave) or on the master.
+
+*** More information about buildslaves via the web plugin (#110)
+
+A new page, rooted at /buildslave/$SLAVENAME, gives extensive information about
+the buildslave.
+
+*** More flexible merging of requests (#415)
+
+The optional c['mergeRequests'] configuration parameter takes a function
+which can decide whether two requests are mergeable.
+
+*** Steps can be made to run even if the build has halted (#414)
+
+Adding alwaysRun=True to a step will cause it to run even if some other step
+has failed and has haltOnFailure=True.
+
+*** Compress buildstep logfiles (#26)
+
+Logs for each buildstep, which can take a lot of space on a busy buildmaster,
+are automatically compressed after the step has finished.
+
+*** Support for "latent" buildslaves
+
+The buildslaves that are started on-demand are called "latent" buildslaves.
+Buildbot ships with an abstract base class for building latent buildslaves, and
+a concrete implementation for AWS EC2. 
+
+*** Customized MailNotifier messages (#175)
+
+MailNotifier now takes an optional function to build the notification message,
+allowing ultimate site-level control over the format of buildbot's notification
+emails.
+
+*** Nightly scheduler support for building only if changes have occurred
+
+With the addition of onlyIfChanged=True, the Nightly scheduler will not schedule
+a new build if no changes have been made since its last scheduled build.
+
+*** Add ATOM/RSS feeds to WebStatus (#372)
+
+Two new pages, /atom and /rss, provide feeds of build events to any feed
+reader.  These paths take the same "category" and "branch" arguments as the
+waterfall and grid.
+
+*** Add categories to Schedulers and Changes (#182)
+
+This allows a moderate amount of support for multiple projects built in a
+single buildmaster.
+
+*** Gracefully shut down a buildslave after its build is complete
+
+The /buildslaves/$SLAVENAME pages have a "Gracefully Shutdown" button which
+will cause the corresponding slave to shut itself down when it finishes its
+current build.  This is a good way to do work on a slave without causing a
+spurious build failure.
+
+*** SVN source steps can send usernames and passwords (#41)
+
+Adding username="foo" and/or password="bar" to an SVN step will cause
+--username and --password arguments to be passed to 'svn' on the slave side.
+Passwords are suitably obfuscated in logfiles.
+
+** New Steps
+
+*** DirectoryUpload (#393)
+
+This step uploads an entire directory to the master, and can be useful when a
+build creates several products (e.g., a client and server package).
+
+*** MasterShellCommand
+
+This step runs a shell command on the server, and can be useful for
+post-processing build products, or performing other maintenance tasks on the
+master.
+
+*** PyLint (#259)
+
+A PyLint step is available to complement the existing PyFlakes step.
+
+** Bugs Fixed
+
+*** Process output from new versions of Test::Harness (#346)
+
+*** Fixes to the try client and scheduler
+
+*** Remove redundant loop in MailNotifier (#315)
+
+*** Display correct $PWD in logfiles (#179)
+
+*** Do not assume a particular python version on Windows (#401)
+
+*** Sort files in changes (#402)
+
+*** Sort buildslaves lexically (#416)
+
+*** Send properties to all builds initiated by AnyBranchScheduler
+
+*** Dependent Schedulers are more robust to reconfiguration (#35)
+
+*** Fix properties handling in triggered buidls (#392)
+
+*** Use "call" on Windows to avoid errors (#417)
+
+*** Support setDefaultWorkdir in FileUpload and FileDownload (#209)
+
+*** Support WithProperties in FileUpload and FileDownload (#210)
+
+*** Fix a bug where changes could be lost on a master crash (#202)
+
+*** Remove color settings from non-presentation code (#251)
+
+*** Fix builders which stopped working after a PING (#349, #85)
+
+*** Isolate Python exceptions in status plugins (#388)
+
+*** Notify about slaves missing at master startup (#302)
+
+*** Fix tracebacks in web display after a reconfig (#176)
+
+** Version-Control Changes
+
+*** Many Mercurial fixes
+
+ - Inrepo branch support finalized (source step + changegroup hook + test case)
+   (#65 #185 #187)
+
+ - Reduced amount of full clones by separating clone with update into
+   clone/pull/update steps (#186, #227) (see #412 for future work here)
+
+ - Fixed mercurial changegroup hook to work with Mercurial 1.1 API (#181, #380)
+
+*** Many git fixes
+
+*** Add got_revision to Perforce support (#127)
+
+*** Use "git foo" everywhere instead of deprecated "git-foo"
+
+** Minor Changes
+
+*** factory.addSteps (#317)
+
+If you have a common list of steps that are included in multiple factories, you
+can use f.addSteps(steplist) to add them all at once.
+
+*** Twisted logfile rotation and cleanup (#108)
+
+By default, Buildbot now rotates and cleans up the (potentially voluminous)
+twistd.log files.
+
+*** Prioritize build requests based on the time they wre submitted (#334)
+
+Balancing of load is a bit more fair, although not true load balancing.
+
 * Release 0.7.9 (15 Sep 2008)
 
 ** New Features
 
 *** Configurable public_html directory (#162)
 
 The public_html/ directory, which provides static content for the WebStatus()
 HTTP server, is now configurable. The default location is still the
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,11 +1,11 @@
 Metadata-Version: 1.0
 Name: buildbot
-Version: 0.7.9
+Version: 0.7.10p1
 Summary: BuildBot build automation system
 Home-page: http://buildbot.net/
 Author: Brian Warner
 Author-email: warner-buildbot@lothar.com
 License: GNU GPL
 Description: 
         The BuildBot is a system to automate the compile/test cycle required by
         most software projects to validate code changes. By automatically
--- a/buildbot.egg-info/PKG-INFO
+++ b/buildbot.egg-info/PKG-INFO
@@ -1,11 +1,11 @@
 Metadata-Version: 1.0
 Name: buildbot
-Version: 0.7.9
+Version: 0.7.10p1
 Summary: BuildBot build automation system
 Home-page: http://buildbot.net/
 Author: Brian Warner
 Author-email: warner-buildbot@lothar.com
 License: GNU GPL
 Description: 
         The BuildBot is a system to automate the compile/test cycle required by
         most software projects to validate code changes. By automatically
--- a/buildbot.egg-info/SOURCES.txt
+++ b/buildbot.egg-info/SOURCES.txt
@@ -6,16 +6,17 @@ README
 README.w32
 setup.py
 bin/buildbot
 buildbot/__init__.py
 buildbot/buildbot.png
 buildbot/buildset.py
 buildbot/buildslave.py
 buildbot/dnotify.py
+buildbot/ec2buildslave.py
 buildbot/interfaces.py
 buildbot/locks.py
 buildbot/manhole.py
 buildbot/master.py
 buildbot/pbutil.py
 buildbot/scheduler.py
 buildbot/sourcestamp.py
 buildbot/util.py
@@ -77,52 +78,64 @@ buildbot/status/words.py
 buildbot/status/web/__init__.py
 buildbot/status/web/about.py
 buildbot/status/web/base.py
 buildbot/status/web/baseweb.py
 buildbot/status/web/build.py
 buildbot/status/web/builder.py
 buildbot/status/web/changes.py
 buildbot/status/web/classic.css
+buildbot/status/web/feeds.py
 buildbot/status/web/grid.py
 buildbot/status/web/index.html
 buildbot/status/web/logs.py
 buildbot/status/web/robots.txt
 buildbot/status/web/slaves.py
 buildbot/status/web/step.py
 buildbot/status/web/tests.py
 buildbot/status/web/waterfall.py
 buildbot/status/web/xmlrpc.py
 buildbot/steps/__init__.py
 buildbot/steps/dummy.py
+buildbot/steps/master.py
 buildbot/steps/maxq.py
 buildbot/steps/python.py
 buildbot/steps/python_twisted.py
 buildbot/steps/shell.py
 buildbot/steps/source.py
 buildbot/steps/transfer.py
 buildbot/steps/trigger.py
+buildbot/steps/package/__init__.py
+buildbot/steps/package/rpm/__init__.py
+buildbot/steps/package/rpm/rpmbuild.py
+buildbot/steps/package/rpm/rpmlint.py
+buildbot/steps/package/rpm/rpmspec.py
 buildbot/test/__init__.py
 buildbot/test/emit.py
 buildbot/test/emitlogs.py
 buildbot/test/runutils.py
 buildbot/test/sleep.py
 buildbot/test/test__versions.py
 buildbot/test/test_bonsaipoller.py
 buildbot/test/test_buildreq.py
 buildbot/test/test_buildstep.py
 buildbot/test/test_changes.py
 buildbot/test/test_config.py
 buildbot/test/test_control.py
 buildbot/test/test_dependencies.py
+buildbot/test/test_ec2buildslave.py
+buildbot/test/test_limitlogs.py
 buildbot/test/test_locks.py
 buildbot/test/test_maildir.py
 buildbot/test/test_mailparse.py
+buildbot/test/test_mergerequests.py
 buildbot/test/test_p4poller.py
+buildbot/test/test_package_rpm.py
 buildbot/test/test_properties.py
+buildbot/test/test_reconfig.py
 buildbot/test/test_run.py
 buildbot/test/test_runner.py
 buildbot/test/test_scheduler.py
 buildbot/test/test_shell.py
 buildbot/test/test_slavecommand.py
 buildbot/test/test_slaves.py
 buildbot/test/test_status.py
 buildbot/test/test_steps.py
@@ -148,18 +161,20 @@ buildbot/test/mail/syncmail.1
 buildbot/test/mail/syncmail.2
 buildbot/test/mail/syncmail.3
 buildbot/test/mail/syncmail.4
 buildbot/test/mail/syncmail.5
 buildbot/test/subdir/emit.py
 contrib/README.txt
 contrib/arch_buildbot.py
 contrib/bb_applet.py
+contrib/bzr_buildbot.py
 contrib/darcs_buildbot.py
 contrib/fakechange.py
+contrib/generate_changelog.py
 contrib/git_buildbot.py
 contrib/hg_buildbot.py
 contrib/run_maxq.py
 contrib/svn_buildbot.py
 contrib/svn_watcher.py
 contrib/svnpoller.py
 contrib/viewcvspoll.py
 contrib/CSS/sample1.css
--- a/buildbot/__init__.py
+++ b/buildbot/__init__.py
@@ -1,2 +1,1 @@
-
-version = "0.7.9"
+version = "0.7.10p1"
--- a/buildbot/buildslave.py
+++ b/buildbot/buildslave.py
@@ -1,24 +1,27 @@
+# Portions copyright Canonical Ltd. 2009
 
 import time
 from email.Message import Message
 from email.Utils import formatdate
 from zope.interface import implements
 from twisted.python import log
 from twisted.internet import defer, reactor
 from twisted.application import service
+import twisted.spread.pb
 
 from buildbot.pbutil import NewCredPerspective
 from buildbot.status.builder import SlaveStatus
 from buildbot.status.mail import MailNotifier
-from buildbot.interfaces import IBuildSlave
+from buildbot.interfaces import IBuildSlave, ILatentBuildSlave
 from buildbot.process.properties import Properties
 
-class BuildSlave(NewCredPerspective, service.MultiService):
+
+class AbstractBuildSlave(NewCredPerspective, service.MultiService):
     """This is the master-side representative for a remote buildbot slave.
     There is exactly one for each slave described in the config file (the
     c['slaves'] list). When buildbots connect in (.attach), they get a
     reference to this instance. The BotMaster object is stashed as the
     .botmaster attribute. The BotMaster is also our '.parent' Service.
 
     I represent a build slave -- a remote machine capable of
     running builds.  I am instantiated by the configuration file, and can be
@@ -31,28 +34,28 @@ class BuildSlave(NewCredPerspective, ser
                  properties={}):
         """
         @param name: botname this machine will supply when it connects
         @param password: password this machine will supply when
                          it connects
         @param max_builds: maximum number of simultaneous builds that will
                            be run concurrently on this buildslave (the
                            default is None for no limit)
-        @param properties: properties that will be applied to builds run on 
+        @param properties: properties that will be applied to builds run on
                            this slave
         @type properties: dictionary
         """
         service.MultiService.__init__(self)
         self.slavename = name
         self.password = password
         self.botmaster = None # no buildmaster yet
         self.slave_status = SlaveStatus(name)
         self.slave = None # a RemoteReference to the Bot, when connected
         self.slave_commands = None
-        self.slavebuilders = []
+        self.slavebuilders = {}
         self.max_builds = max_builds
 
         self.properties = Properties()
         self.properties.update(properties, "BuildSlave")
         self.properties.setProperty("slavename", name, "BuildSlave")
 
         self.lastMessageReceived = 0
         if isinstance(notify_on_missing, str):
@@ -73,33 +76,73 @@ class BuildSlave(NewCredPerspective, ser
         assert self.slavename == new.slavename
         assert self.password == new.password
         assert self.__class__ == new.__class__
         self.max_builds = new.max_builds
 
     def __repr__(self):
         if self.botmaster:
             builders = self.botmaster.getBuildersForSlave(self.slavename)
-            return "<BuildSlave '%s', current builders: %s>" % \
-               (self.slavename, ','.join(map(lambda b: b.name, builders)))
+            return "<%s '%s', current builders: %s>" % \
+               (self.__class__.__name__, self.slavename,
+                ','.join(map(lambda b: b.name, builders)))
         else:
-            return "<BuildSlave '%s', (no builders yet)>" % self.slavename
+            return "<%s '%s', (no builders yet)>" % \
+                (self.__class__.__name__, self.slavename)
 
     def setBotmaster(self, botmaster):
         assert not self.botmaster, "BuildSlave already has a botmaster"
         self.botmaster = botmaster
+        self.startMissingTimer()
+
+    def stopMissingTimer(self):
+        if self.missing_timer:
+            self.missing_timer.cancel()
+            self.missing_timer = None
+
+    def startMissingTimer(self):
+        if self.notify_on_missing and self.missing_timeout and self.parent:
+            self.stopMissingTimer() # in case it's already running
+            self.missing_timer = reactor.callLater(self.missing_timeout,
+                                                   self._missing_timer_fired)
+
+    def _missing_timer_fired(self):
+        self.missing_timer = None
+        # notify people, but only if we're still in the config
+        if not self.parent:
+            return
+
+        buildmaster = self.botmaster.parent
+        status = buildmaster.getStatus()
+        text = "The Buildbot working for '%s'\n" % status.getProjectName()
+        text += ("has noticed that the buildslave named %s went away\n" %
+                 self.slavename)
+        text += "\n"
+        text += ("It last disconnected at %s (buildmaster-local time)\n" %
+                 time.ctime(time.time() - self.missing_timeout)) # approx
+        text += "\n"
+        text += "The admin on record (as reported by BUILDSLAVE:info/admin)\n"
+        text += "was '%s'.\n" % self.slave_status.getAdmin()
+        text += "\n"
+        text += "Sincerely,\n"
+        text += " The Buildbot\n"
+        text += " %s\n" % status.getProjectURL()
+        subject = "Buildbot: buildslave %s was lost" % self.slavename
+        return self._mail_missing_message(subject, text)
+
 
     def updateSlave(self):
         """Called to add or remove builders after the slave has connected.
 
         @return: a Deferred that indicates when an attached slave has
         accepted the new builders and/or released the old ones."""
         if self.slave:
             return self.sendBuilderList()
-        return defer.succeed(None)
+        else:
+            return defer.succeed(None)
 
     def updateSlaveStatus(self, buildStarted=None, buildFinished=None):
         if buildStarted:
             self.slave_status.buildStarted(buildStarted)
         if buildFinished:
             self.slave_status.buildFinished(buildFinished)
 
     def attached(self, bot):
@@ -128,16 +171,21 @@ class BuildSlave(NewCredPerspective, ser
         # now we go through a sequence of calls, gathering information, then
         # tell the Botmaster that it can finally give this slave to all the
         # Builders that care about it.
 
         # we accumulate slave information in this 'state' dictionary, then
         # set it atomically if we make it far enough through the process
         state = {}
 
+        # Reset graceful shutdown status
+        self.slave_status.setGraceful(False)
+        # We want to know when the graceful shutdown flag changes
+        self.slave_status.addGracefulWatcher(self._gracefulChanged)
+
         def _log_attachment_on_slave(res):
             d1 = bot.callRemote("print", "attached")
             d1.addErrback(lambda why: None)
             return d1
         d.addCallback(_log_attachment_on_slave)
 
         def _get_info(res):
             d1 = bot.callRemote("getSlaveInfo")
@@ -171,85 +219,36 @@ class BuildSlave(NewCredPerspective, ser
         def _accept_slave(res):
             self.slave_status.setAdmin(state.get("admin"))
             self.slave_status.setHost(state.get("host"))
             self.slave_status.setConnected(True)
             self.slave_commands = state.get("slave_commands")
             self.slave = bot
             log.msg("bot attached")
             self.messageReceivedFromSlave()
-            if self.missing_timer:
-                self.missing_timer.cancel()
-                self.missing_timer = None
+            self.stopMissingTimer()
 
             return self.updateSlave()
         d.addCallback(_accept_slave)
 
         # Finally, the slave gets a reference to this BuildSlave. They
         # receive this later, after we've started using them.
         d.addCallback(lambda res: self)
         return d
 
     def messageReceivedFromSlave(self):
         now = time.time()
         self.lastMessageReceived = now
         self.slave_status.setLastMessageReceived(now)
 
     def detached(self, mind):
         self.slave = None
+        self.slave_status.removeGracefulWatcher(self._gracefulChanged)
         self.slave_status.setConnected(False)
-        self.botmaster.slaveLost(self)
         log.msg("BuildSlave.detached(%s)" % self.slavename)
-        if self.notify_on_missing and self.parent and not self.missing_timer:
-            self.missing_timer = reactor.callLater(self.missing_timeout,
-                                                   self._missing_timer_fired)
-
-    def _missing_timer_fired(self):
-        self.missing_timer = None
-        # notify people, but only if we're still in the config
-        if not self.parent:
-            return
-
-        # first, see if we have a MailNotifier we can use. This gives us a
-        # fromaddr and a relayhost.
-        buildmaster = self.botmaster.parent
-        status = buildmaster.getStatus()
-        for st in buildmaster.statusTargets:
-            if isinstance(st, MailNotifier):
-                break
-        else:
-            # if not, they get a default MailNotifier, which always uses SMTP
-            # to localhost and uses a dummy fromaddr of "buildbot".
-            log.msg("buildslave-missing msg using default MailNotifier")
-            st = MailNotifier("buildbot")
-        # now construct the mail
-        text = "The Buildbot working for '%s'\n" % status.getProjectName()
-        text += ("has noticed that the buildslave named %s went away\n" %
-                 self.slavename)
-        text += "\n"
-        text += ("It last disconnected at %s (buildmaster-local time)\n" %
-                 time.ctime(time.time() - self.missing_timeout)) # close enough
-        text += "\n"
-        text += "The admin on record (as reported by BUILDSLAVE:info/admin)\n"
-        text += "was '%s'.\n" % self.slave_status.getAdmin()
-        text += "\n"
-        text += "Sincerely,\n"
-        text += " The Buildbot\n"
-        text += " %s\n" % status.getProjectURL()
-
-        m = Message()
-        m.set_payload(text)
-        m['Date'] = formatdate(localtime=True)
-        m['Subject'] = "Buildbot: buildslave %s was lost" % self.slavename
-        m['From'] = st.fromaddr
-        recipients = self.notify_on_missing
-        m['To'] = ", ".join(recipients)
-        d = st.sendMessage(m, recipients)
-        # return the Deferred for testing purposes
-        return d
 
     def disconnect(self):
         """Forcibly disconnect the slave.
 
         This severs the TCP connection and returns a Deferred that will fire
         (with None) when the connection is probably gone.
 
         If the slave is still alive, they will probably try to reconnect
@@ -260,58 +259,166 @@ class BuildSlave(NewCredPerspective, ser
         reconnect, they will be rejected as an unknown slave. The second is
         when we wind up with two connections for the same slave, in which
         case we disconnect the older connection.
         """
 
         if not self.slave:
             return defer.succeed(None)
         log.msg("disconnecting old slave %s now" % self.slavename)
+        # When this Deferred fires, we'll be ready to accept the new slave
+        return self._disconnect(self.slave)
 
+    def _disconnect(self, slave):
         # all kinds of teardown will happen as a result of
         # loseConnection(), but it happens after a reactor iteration or
         # two. Hook the actual disconnect so we can know when it is safe
         # to connect the new slave. We have to wait one additional
         # iteration (with callLater(0)) to make sure the *other*
         # notifyOnDisconnect handlers have had a chance to run.
         d = defer.Deferred()
 
         # notifyOnDisconnect runs the callback with one argument, the
         # RemoteReference being disconnected.
         def _disconnected(rref):
             reactor.callLater(0, d.callback, None)
-        self.slave.notifyOnDisconnect(_disconnected)
-        tport = self.slave.broker.transport
+        slave.notifyOnDisconnect(_disconnected)
+        tport = slave.broker.transport
         # this is the polite way to request that a socket be closed
         tport.loseConnection()
         try:
             # but really we don't want to wait for the transmit queue to
             # drain. The remote end is unlikely to ACK the data, so we'd
             # probably have to wait for a (20-minute) TCP timeout.
             #tport._closeSocket()
             # however, doing _closeSocket (whether before or after
             # loseConnection) somehow prevents the notifyOnDisconnect
             # handlers from being run. Bummer.
             tport.offset = 0
             tport.dataBuffer = ""
-            pass
         except:
             # however, these hacks are pretty internal, so don't blow up if
             # they fail or are unavailable
             log.msg("failed to accelerate the shutdown process")
             pass
         log.msg("waiting for slave to finish disconnecting")
 
-        # When this Deferred fires, we'll be ready to accept the new slave
         return d
 
     def sendBuilderList(self):
         our_builders = self.botmaster.getBuildersForSlave(self.slavename)
         blist = [(b.name, b.builddir) for b in our_builders]
         d = self.slave.callRemote("setBuilderList", blist)
+        return d
+
+    def perspective_keepalive(self):
+        pass
+
+    def addSlaveBuilder(self, sb):
+        if sb.builder_name not in self.slavebuilders:
+            log.msg("%s adding %s" % (self, sb))
+        elif sb is not self.slavebuilders[sb.builder_name]:
+            log.msg("%s replacing %s" % (self, sb))
+        else:
+            return
+        self.slavebuilders[sb.builder_name] = sb
+
+    def removeSlaveBuilder(self, sb):
+        try:
+            del self.slavebuilders[sb.builder_name]
+        except KeyError:
+            pass
+        else:
+            log.msg("%s removed %s" % (self, sb))
+
+    def canStartBuild(self):
+        """
+        I am called when a build is requested to see if this buildslave
+        can start a build.  This function can be used to limit overall
+        concurrency on the buildslave.
+        """
+        # If we're waiting to shutdown gracefully, then we shouldn't
+        # accept any new jobs.
+        if self.slave_status.getGraceful():
+            return False
+
+        if self.max_builds:
+            active_builders = [sb for sb in self.slavebuilders.values()
+                               if sb.isBusy()]
+            if len(active_builders) >= self.max_builds:
+                return False
+        return True
+
+    def _mail_missing_message(self, subject, text):
+        # first, see if we have a MailNotifier we can use. This gives us a
+        # fromaddr and a relayhost.
+        buildmaster = self.botmaster.parent
+        for st in buildmaster.statusTargets:
+            if isinstance(st, MailNotifier):
+                break
+        else:
+            # if not, they get a default MailNotifier, which always uses SMTP
+            # to localhost and uses a dummy fromaddr of "buildbot".
+            log.msg("buildslave-missing msg using default MailNotifier")
+            st = MailNotifier("buildbot")
+        # now construct the mail
+
+        m = Message()
+        m.set_payload(text)
+        m['Date'] = formatdate(localtime=True)
+        m['Subject'] = subject
+        m['From'] = st.fromaddr
+        recipients = self.notify_on_missing
+        m['To'] = ", ".join(recipients)
+        d = st.sendMessage(m, recipients)
+        # return the Deferred for testing purposes
+        return d
+
+    def _gracefulChanged(self, graceful):
+        """This is called when our graceful shutdown setting changes"""
+        if graceful:
+            active_builders = [sb for sb in self.slavebuilders.values()
+                               if sb.isBusy()]
+            if len(active_builders) == 0:
+                # Shut down!
+                self.shutdown()
+
+    def shutdown(self):
+        """Shutdown the slave"""
+        # Look for a builder with a remote reference to the client side
+        # slave.  If we can find one, then call "shutdown" on the remote
+        # builder, which will cause the slave buildbot process to exit.
+        d = None
+        for b in self.slavebuilders.values():
+            if b.remote:
+                d = b.remote.callRemote("shutdown")
+                break
+
+        if d:
+            log.msg("Shutting down slave: %s" % self.slavename)
+            # The remote shutdown call will not complete successfully since the
+            # buildbot process exits almost immediately after getting the
+            # shutdown request.
+            # Here we look at the reason why the remote call failed, and if
+            # it's because the connection was lost, that means the slave
+            # shutdown as expected.
+            def _errback(why):
+                if why.check(twisted.spread.pb.PBConnectionLost):
+                    log.msg("Lost connection to %s" % self.slavename)
+                else:
+                    log.err("Unexpected error when trying to shutdown %s" % self.slavename)
+            d.addErrback(_errback)
+            return d
+        log.err("Couldn't find remote builder to shut down slave")
+        return defer.succeed(None)
+
+class BuildSlave(AbstractBuildSlave):
+
+    def sendBuilderList(self):
+        d = AbstractBuildSlave.sendBuilderList(self)
         def _sent(slist):
             dl = []
             for name, remote in slist.items():
                 # use get() since we might have changed our mind since then
                 b = self.botmaster.builders.get(name)
                 if b:
                     d1 = b.attached(self, remote, self.slave_commands)
                     dl.append(d1)
@@ -319,32 +426,263 @@ class BuildSlave(NewCredPerspective, ser
         def _set_failed(why):
             log.msg("BuildSlave.sendBuilderList (%s) failed" % self)
             log.err(why)
             # TODO: hang up on them?, without setBuilderList we can't use
             # them
         d.addCallbacks(_sent, _set_failed)
         return d
 
-    def perspective_keepalive(self):
-        pass
+    def detached(self, mind):
+        AbstractBuildSlave.detached(self, mind)
+        self.botmaster.slaveLost(self)
+        self.startMissingTimer()
+
+    def buildFinished(self, sb):
+        """This is called when a build on this slave is finished."""
+        # If we're gracefully shutting down, and we have no more active
+        # builders, then it's safe to disconnect
+        if self.slave_status.getGraceful():
+            active_builders = [sb for sb in self.slavebuilders.values()
+                               if sb.isBusy()]
+            if len(active_builders) == 0:
+                # Shut down!
+                return self.shutdown()
+        return defer.succeed(None)
+
+class AbstractLatentBuildSlave(AbstractBuildSlave):
+    """A build slave that will start up a slave instance when needed.
+
+    To use, subclass and implement start_instance and stop_instance.
+
+    See ec2buildslave.py for a concrete example.  Also see the stub example in
+    test/test_slaves.py.
+    """
+
+    implements(ILatentBuildSlave)
+
+    substantiated = False
+    substantiation_deferred = None
+    build_wait_timer = None
+    _start_result = _shutdown_callback_handle = None
+
+    def __init__(self, name, password, max_builds=None,
+                 notify_on_missing=[], missing_timeout=60*20,
+                 build_wait_timeout=60*10,
+                 properties={}):
+        AbstractBuildSlave.__init__(
+            self, name, password, max_builds, notify_on_missing,
+            missing_timeout, properties)
+        self.building = set()
+        self.build_wait_timeout = build_wait_timeout
+
+    def start_instance(self):
+        # responsible for starting instance that will try to connect with
+        # this master.  Should return deferred.  Problems should use an
+        # errback.
+        raise NotImplementedError
+
+    def stop_instance(self, fast=False):
+        # responsible for shutting down instance.
+        raise NotImplementedError
 
-    def addSlaveBuilder(self, sb):
-        log.msg("%s adding %s" % (self, sb))
-        self.slavebuilders.append(sb)
+    def substantiate(self, sb):
+        if self.substantiated:
+            self._clearBuildWaitTimer()
+            self._setBuildWaitTimer()
+            return defer.succeed(self)
+        if self.substantiation_deferred is None:
+            if self.parent and not self.missing_timer:
+                # start timer.  if timer times out, fail deferred
+                self.missing_timer = reactor.callLater(
+                    self.missing_timeout,
+                    self._substantiation_failed, defer.TimeoutError())
+            self.substantiation_deferred = defer.Deferred()
+            if self.slave is None:
+                self._substantiate() # start up instance
+            # else: we're waiting for an old one to detach.  the _substantiate
+            # will be done in ``detached`` below.
+        return self.substantiation_deferred
+
+    def _substantiate(self):
+        # register event trigger
+        d = self.start_instance()
+        self._shutdown_callback_handle = reactor.addSystemEventTrigger(
+            'before', 'shutdown', self._soft_disconnect, fast=True)
+        def stash_reply(result):
+            self._start_result = result
+        def clean_up(failure):
+            if self.missing_timer is not None:
+                self.missing_timer.cancel()
+                self._substantiation_failed(failure)
+            if self._shutdown_callback_handle is not None:
+                handle = self._shutdown_callback_handle
+                del self._shutdown_callback_handle
+                reactor.removeSystemEventTrigger(handle)
+            return failure
+        d.addCallbacks(stash_reply, clean_up)
+        return d
 
-    def removeSlaveBuilder(self, sb):
-        log.msg("%s removing %s" % (self, sb))
-        if sb in self.slavebuilders:
-            self.slavebuilders.remove(sb)
+    def attached(self, bot):
+        if self.substantiation_deferred is None:
+            log.msg('Slave %s received connection while not trying to '
+                    'substantiate.  Disconnecting.' % (self.slavename,))
+            self._disconnect(bot)
+            return defer.fail()
+        return AbstractBuildSlave.attached(self, bot)
+
+    def detached(self, mind):
+        AbstractBuildSlave.detached(self, mind)
+        if self.substantiation_deferred is not None:
+            self._substantiate()
+
+    def _substantiation_failed(self, failure):
+        d = self.substantiation_deferred
+        self.substantiation_deferred = None
+        self.missing_timer = None
+        d.errback(failure)
+        self.insubstantiate()
+        # notify people, but only if we're still in the config
+        if not self.parent or not self.notify_on_missing:
+            return
+
+        status = buildmaster.getStatus()
+        text = "The Buildbot working for '%s'\n" % status.getProjectName()
+        text += ("has noticed that the latent buildslave named %s \n" %
+                 self.slavename)
+        text += "never substantiated after a request\n"
+        text += "\n"
+        text += ("The request was made at %s (buildmaster-local time)\n" %
+                 time.ctime(time.time() - self.missing_timeout)) # approx
+        text += "\n"
+        text += "Sincerely,\n"
+        text += " The Buildbot\n"
+        text += " %s\n" % status.getProjectURL()
+        subject = "Buildbot: buildslave %s never substantiated" % self.slavename
+        return self._mail_missing_message(subject, text)
 
-    def canStartBuild(self):
-        """
-        I am called when a build is requested to see if this buildslave
-        can start a build.  This function can be used to limit overall
-        concurrency on the buildslave.
-        """
-        if self.max_builds:
-            active_builders = [sb for sb in self.slavebuilders if sb.isBusy()]
-            if len(active_builders) >= self.max_builds:
-                return False
-        return True
+    def buildStarted(self, sb):
+        assert self.substantiated
+        self._clearBuildWaitTimer()
+        self.building.add(sb.builder_name)
+
+    def buildFinished(self, sb):
+        self.building.remove(sb.builder_name)
+        if not self.building:
+            self._setBuildWaitTimer()
+
+    def _clearBuildWaitTimer(self):
+        if self.build_wait_timer is not None:
+            if self.build_wait_timer.active():
+                self.build_wait_timer.cancel()
+            self.build_wait_timer = None
+
+    def _setBuildWaitTimer(self):
+        self._clearBuildWaitTimer()
+        self.build_wait_timer = reactor.callLater(
+            self.build_wait_timeout, self._soft_disconnect)
+
+    def insubstantiate(self, fast=False):
+        self._clearBuildWaitTimer()
+        d = self.stop_instance(fast)
+        if self._shutdown_callback_handle is not None:
+            handle = self._shutdown_callback_handle
+            del self._shutdown_callback_handle
+            reactor.removeSystemEventTrigger(handle)
+        self.substantiated = False
+        self.building.clear() # just to be sure
+        return d
+
+    def _soft_disconnect(self, fast=False):
+        d = AbstractBuildSlave.disconnect(self)
+        if self.slave is not None:
+            # this could be called when the slave needs to shut down, such as
+            # in BotMaster.removeSlave, *or* when a new slave requests a
+            # connection when we already have a slave. It's not clear what to
+            # do in the second case: this shouldn't happen, and if it
+            # does...if it's a latent slave, shutting down will probably kill
+            # something we want...but we can't know what the status is. So,
+            # here, we just do what should be appropriate for the first case,
+            # and put our heads in the sand for the second, at least for now.
+            # The best solution to the odd situation is removing it as a
+            # possibilty: make the master in charge of connecting to the
+            # slave, rather than vice versa. TODO.
+            d = defer.DeferredList([d, self.insubstantiate(fast)])
+        else:
+            if self.substantiation_deferred is not None:
+                # unlike the previous block, we don't expect this situation when
+                # ``attached`` calls ``disconnect``, only when we get a simple
+                # request to "go away".
+                self.substantiation_deferred.errback()
+                self.substantiation_deferred = None
+                if self.missing_timer:
+                    self.missing_timer.cancel()
+                    self.missing_timer = None
+                self.stop_instance()
+        return d
 
+    def disconnect(self):
+        d = self._soft_disconnect()
+        # this removes the slave from all builders.  It won't come back
+        # without a restart (or maybe a sighup)
+        self.botmaster.slaveLost(self)
+
+    def stopService(self):
+        res = defer.maybeDeferred(AbstractBuildSlave.stopService, self)
+        if self.slave is not None:
+            d = self._soft_disconnect()
+            res = defer.DeferredList([res, d])
+        return res
+
+    def updateSlave(self):
+        """Called to add or remove builders after the slave has connected.
+
+        Also called after botmaster's builders are initially set.
+
+        @return: a Deferred that indicates when an attached slave has
+        accepted the new builders and/or released the old ones."""
+        for b in self.botmaster.getBuildersForSlave(self.slavename):
+            if b.name not in self.slavebuilders:
+                b.addLatentSlave(self)
+        return AbstractBuildSlave.updateSlave(self)
+
+    def sendBuilderList(self):
+        d = AbstractBuildSlave.sendBuilderList(self)
+        def _sent(slist):
+            dl = []
+            for name, remote in slist.items():
+                # use get() since we might have changed our mind since then.
+                # we're checking on the builder in addition to the
+                # slavebuilders out of a bit of paranoia.
+                b = self.botmaster.builders.get(name)
+                sb = self.slavebuilders.get(name)
+                if b and sb:
+                    d1 = sb.attached(self, remote, self.slave_commands)
+                    dl.append(d1)
+            return defer.DeferredList(dl)
+        def _set_failed(why):
+            log.msg("BuildSlave.sendBuilderList (%s) failed" % self)
+            log.err(why)
+            # TODO: hang up on them?, without setBuilderList we can't use
+            # them
+            if self.substantiation_deferred:
+                self.substantiation_deferred.errback()
+                self.substantiation_deferred = None
+            if self.missing_timer:
+                self.missing_timer.cancel()
+                self.missing_timer = None
+            # TODO: maybe log?  send an email?
+            return why
+        d.addCallbacks(_sent, _set_failed)
+        def _substantiated(res):
+            self.substantiated = True
+            if self.substantiation_deferred:
+                d = self.substantiation_deferred
+                del self.substantiation_deferred
+                res = self._start_result
+                del self._start_result
+                d.callback(res)
+            # note that the missing_timer is already handled within
+            # ``attached``
+            if not self.building:
+                self._setBuildWaitTimer()
+        d.addCallback(_substantiated)
+        return d
--- a/buildbot/changes/changes.py
+++ b/buildbot/changes/changes.py
@@ -49,27 +49,31 @@ class Change:
 
     number = None
 
     links = []
     branch = None
     revision = None # used to create a source-stamp
 
     def __init__(self, who, files, comments, isdir=0, links=[],
-                 revision=None, when=None, branch=None):
+                 revision=None, when=None, branch=None, category=None):
         self.who = who
-        self.files = files
         self.comments = comments
         self.isdir = isdir
         self.links = links
         self.revision = revision
         if when is None:
             when = util.now()
         self.when = when
         self.branch = branch
+        self.category = category
+
+        # keep a sorted list of the files, for easier display
+        self.files = files[:]
+        self.files.sort()
 
     def asText(self):
         data = ""
         data += self.getFileContents() 
         data += "At: %s\n" % self.getTime()
         data += "Changed By: %s\n" % self.who
         data += "Comments: %s\n\n" % self.comments
         return data
@@ -105,36 +109,38 @@ class Change:
         using our asHTML method. The Change is free to use this or ignore it
         as it pleases.
 
         @return: the HTML that will be put inside the table cell. Typically
         this is just a single href named after the author of the change and
         pointing at the passed-in 'url'.
         """
         who = self.getShortAuthor()
+        if self.comments is None:
+            title = ""
+        else:
+            title = html.escape(self.comments)
         return '<a href="%s" title="%s">%s</a>' % (url,
-                                                   html.escape(self.comments),
+                                                   title,
                                                    html.escape(who))
 
     def getShortAuthor(self):
         return self.who
 
     def getTime(self):
         if not self.when:
             return "?"
         return time.strftime("%a %d %b %Y %H:%M:%S",
                              time.localtime(self.when))
 
     def getTimes(self):
         return (self.when, None)
 
     def getText(self):
         return [html.escape(self.who)]
-    def getColor(self):
-        return "white"
     def getLogs(self):
         return {}
 
     def getFileContents(self):
         data = ""
         if len(self.files) == 1:
             if self.isdir:
                 data += "Directory: %s\n" % self.files[0]
@@ -204,19 +210,19 @@ class ChangeMaster(service.MultiService)
             print "ChangeMaster.removeSource", source, source.parent
         d = defer.maybeDeferred(source.disownServiceParent)
         return d
 
     def addChange(self, change):
         """Deliver a file change event. The event should be a Change object.
         This method will timestamp the object as it is received."""
         log.msg("adding change, who %s, %d files, rev=%s, branch=%s, "
-                "comments %s" % (change.who, len(change.files),
-                                 change.revision, change.branch,
-                                 change.comments))
+                "comments %s, category %s" % (change.who, len(change.files),
+                                              change.revision, change.branch,
+                                              change.comments, change.category))
         change.number = self.nextNumber
         self.nextNumber += 1
         self.changes.append(change)
         self.parent.addChange(change)
         # TODO: call pruneChanges after a while
 
     def pruneChanges(self):
         self.changes = self.changes[-100:] # or something
@@ -270,8 +276,13 @@ class ChangeMaster(service.MultiService)
             os.rename(tmpfilename, filename)
         except Exception, e:
             log.msg("unable to save changes")
             log.err()
 
     def stopService(self):
         self.saveYourself()
         return service.MultiService.stopService(self)
+
+class TestChangeMaster(ChangeMaster):
+    """A ChangeMaster for use in tests that does not save itself"""
+    def stopService(self):
+        return service.MultiService.stopService(self)
--- a/buildbot/changes/hgbuildbot.py
+++ b/buildbot/changes/hgbuildbot.py
@@ -29,17 +29,18 @@
 #                                        #
 #                                        # inrepo:  branch = mercurial branch
 #
 #   branch = branchname                  # if set, branch is always branchname
 
 import os
 
 from mercurial.i18n import gettext as _
-from mercurial.node import bin, hex
+from mercurial.node import bin, hex, nullid
+from mercurial.context import workingctx
 
 # mercurial's on-demand-importing hacks interfere with the:
 #from zope.interface import Interface
 # that Twisted needs to do, so disable it.
 try:
     from mercurial import demandimport
     demandimport.disable()
 except ImportError:
@@ -60,45 +61,51 @@ def hook(ui, repo, hooktype, node=None, 
                  "order to use buildbot hook\n")
         return
 
     if branch is None:
         if branchtype is not None:
             if branchtype == 'dirname':
                 branch = os.path.basename(os.getcwd())
             if branchtype == 'inrepo':
-                branch=repo.workingctx().branch()
+                branch = workingctx(repo).branch()
 
     if hooktype == 'changegroup':
         s = sendchange.Sender(master, None)
         d = defer.Deferred()
         reactor.callLater(0, d.callback, None)
         # process changesets
         def _send(res, c):
             ui.status("rev %s sent\n" % c['revision'])
             return s.send(c['branch'], c['revision'], c['comments'],
                           c['files'], c['username'])
 
-        node=bin(node)
-        start = repo.changelog.rev(node)
-        end = repo.changelog.count()
+        try:    # first try Mercurial 1.1+ api
+            start = repo[node].rev()
+            end = len(repo)
+        except TypeError:   # else fall back to old api
+            start = repo.changelog.rev(bin(node))
+            end = repo.changelog.count()
+
         for rev in xrange(start, end):
             # send changeset
-            n = repo.changelog.node(rev)
-            changeset=repo.changelog.read(n)
+            node = repo.changelog.node(rev)
+            manifest, user, (time, timezone), files, desc, extra = repo.changelog.read(node)
+            parents = filter(lambda p: not p == nullid, repo.changelog.parents(node))
+            if branchtype == 'inrepo':
+                branch = extra['branch']
+            # merges don't always contain files, but at least one file is required by buildbot
+            if len(parents) > 1 and not files:
+                files = ["merge"]
             change = {
                 'master': master,
-                # note: this is more likely to be a full email address, which
-                # would make the left-hand "Changes" column kind of wide. The
-                # buildmaster should probably be improved to display an
-                # abbreviation of the username.
-                'username': changeset[1],
-                'revision': hex(n),
-                'comments': changeset[4],
-                'files': changeset[3],
+                'username': user,
+                'revision': hex(node),
+                'comments': desc,
+                'files': files,
                 'branch': branch
             }
             d.addCallback(_send, change)
 
         d.addCallbacks(s.printSuccess, s.printFailure)
         d.addBoth(s.stop)
         s.run()
     else:
--- a/buildbot/changes/pb.py
+++ b/buildbot/changes/pb.py
@@ -29,31 +29,32 @@ class ChangePerspective(NewCredPerspecti
             pathnames.append(path)
 
         if pathnames:
             change = changes.Change(changedict['who'],
                                     pathnames,
                                     changedict['comments'],
                                     branch=changedict.get('branch'),
                                     revision=changedict.get('revision'),
+                                    category=changedict.get('category'),
                                     )
             self.changemaster.addChange(change)
 
 class PBChangeSource(base.ChangeSource):
     compare_attrs = ["user", "passwd", "port", "prefix"]
 
     def __init__(self, user="change", passwd="changepw", port=None,
                  prefix=None, sep=None):
         """I listen on a TCP port for Changes from 'buildbot sendchange'.
 
         I am a ChangeSource which will accept Changes from a remote source. I
         share a TCP listening port with the buildslaves.
 
-        Both the 'buildbot sendchange' command and the
-        contrib/svn_buildbot.py tool know how to send changes to me.
+        The 'buildbot sendchange' command, the contrib/svn_buildbot.py tool,
+        and the contrib/bzr_buildbot.py tool know how to send changes to me.
 
         @type prefix: string (or None)
         @param prefix: if set, I will ignore any filenames that do not start
                        with this string. Moreover I will remove this string
                        from all filenames before creating the Change object
                        and delivering it to the Schedulers. This is useful
                        for changes coming from version control systems that
                        represent branches as parent directories within the
@@ -100,9 +101,8 @@ class PBChangeSource(base.ChangeSource):
     def stopService(self):
         base.ChangeSource.stopService(self)
         # unregister our username
         master = self.parent.parent
         master.dispatcher.unregister(self.user)
 
     def getPerspective(self):
         return ChangePerspective(self.parent, self.prefix)
-
--- a/buildbot/changes/svnpoller.py
+++ b/buildbot/changes/svnpoller.py
@@ -360,18 +360,21 @@ class SVNPoller(base.ChangeSource, util.
                                                        self.last_change)
         self.last_change = new_last_change
         log.msg('svnPoller: _process_changes %s .. %s' %
                 (last_change, new_last_change))
         return new_logentries
 
 
     def _get_text(self, element, tag_name):
-        child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
-        text = "".join([t.data for t in child_nodes])
+        try:
+            child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
+            text = "".join([t.data for t in child_nodes])
+        except:
+            text = "<unknown>"
         return text
 
     def _transform_path(self, path):
         _assert(path.startswith(self._prefix),
                 "filepath '%s' should start with prefix '%s'" %
                 (path, self._prefix))
         relative_path = path[len(self._prefix):]
         if relative_path.startswith("/"):
--- a/buildbot/clients/gtkPanes.py
+++ b/buildbot/clients/gtkPanes.py
@@ -10,16 +10,18 @@ import gobject, gtk
 assert(gtk.Window) # in gtk1 it's gtk.GtkWindow
 
 from twisted.spread import pb
 
 #from buildbot.clients.base import Builder, Client
 from buildbot.clients.base import TextClient
 from buildbot.util import now
 
+from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
+
 '''
 class Pane:
     def __init__(self):
         pass
 
 class OneRow(Pane):
     """This is a one-row status bar. It has one square per Builder, and that
     square is either red, yellow, or green. """
@@ -323,22 +325,29 @@ class ThreeRowBuilder:
         return self.last.getBox(), self.current.getBox(), self.step.getBox()
 
     def getLastBuild(self):
         d = self.ref.callRemote("getLastFinishedBuild")
         d.addCallback(self.gotLastBuild)
     def gotLastBuild(self, build):
         if build:
             build.callRemote("getText").addCallback(self.gotLastText)
-            build.callRemote("getColor").addCallback(self.gotLastColor)
+            build.callRemote("getResults").addCallback(self.gotLastResult)
 
     def gotLastText(self, text):
+        print "Got text", text
         self.last.setText("\n".join(text))
-    def gotLastColor(self, color):
-        self.last.setColor(color)
+
+    def gotLastResult(self, result):
+        colormap = {SUCCESS: 'green',
+                    FAILURE: 'red',
+                    WARNINGS: 'orange',
+                    EXCEPTION: 'purple',
+                    }
+        self.last.setColor(colormap[result])
 
     def getState(self):
         self.ref.callRemote("getState").addCallback(self.gotState)
     def gotState(self, res):
         state, ETA, builds = res
         # state is one of: offline, idle, waiting, interlocked, building
         # TODO: ETA is going away, you have to look inside the builds to get
         # that value
--- a/buildbot/clients/sendchange.py
+++ b/buildbot/clients/sendchange.py
@@ -5,21 +5,21 @@ from twisted.internet import reactor
 
 class Sender:
     def __init__(self, master, user=None):
         self.user = user
         self.host, self.port = master.split(":")
         self.port = int(self.port)
         self.num_changes = 0
 
-    def send(self, branch, revision, comments, files, user=None):
+    def send(self, branch, revision, comments, files, user=None, category=None):
         if user is None:
             user = self.user
         change = {'who': user, 'files': files, 'comments': comments,
-                  'branch': branch, 'revision': revision}
+                  'branch': branch, 'revision': revision, 'category': category}
         self.num_changes += 1
 
         f = pb.PBClientFactory()
         d = f.login(credentials.UsernamePassword("change", "changepw"))
         reactor.connectTCP(self.host, self.port, f)
         d.addCallback(self.addChange, change)
         return d
 
--- a/buildbot/interfaces.py
+++ b/buildbot/interfaces.py
@@ -1,23 +1,25 @@
 
 """Interface documentation.
 
 Define the interfaces that are implemented by various buildbot classes.
 """
 
-from zope.interface import Interface
+from zope.interface import Interface, Attribute
 
 # exceptions that can be raised while trying to start a build
 class NoSlaveError(Exception):
     pass
 class BuilderInUseError(Exception):
     pass
 class BuildSlaveTooOldError(Exception):
     pass
+class LatentBuildSlaveFailedToSubstantiate(Exception):
+    pass
 
 # other exceptions
 class BuildbotNotRunningError(Exception):
     pass
 
 class IChangeSource(Interface):
     """Object which feeds Change objects to the changemaster. When files or
     directories are changed and the version control system provides some
@@ -38,17 +40,17 @@ class IChangeSource(Interface):
     def describe():
         """Should return a string which briefly describes this source. This
         string will be displayed in an HTML status page."""
 
 class IScheduler(Interface):
     """I watch for Changes in the source tree and decide when to trigger
     Builds. I create BuildSet objects and submit them to the BuildMaster. I
     am a service, and the BuildMaster is always my parent.
-    
+
     @ivar properties: properties to be applied to all builds started by this
     scheduler
     @type properties: L<buildbot.process.properties.Properties>
     """
 
     def addChange(change):
         """A Change has just been dispatched by one of the ChangeSources.
         Each Scheduler will receive this Change. I may decide to start a
@@ -75,16 +77,26 @@ class IUpstreamScheduler(Interface):
         """Request that the target callbable be invoked after every
         successful buildset. The target will be called with a single
         argument: the SourceStamp used by the successful builds."""
 
     def listBuilderNames():
         """Return a list of strings indicating the Builders that this
         Scheduler might feed."""
 
+class IDownstreamScheduler(Interface):
+    """This marks an IScheduler to be listening to other schedulers.
+    On reconfigs, these might get notified to check if their upstream
+    scheduler are stil the same."""
+
+    def checkUpstreamScheduler():
+        """Check if the upstream scheduler is still alive, and if not,
+        get a new upstream object from the master."""
+
+
 class ISourceStamp(Interface):
     """
     @cvar branch: branch from which source was drawn
     @type branch: string or None
 
     @cvar revision: revision of the source, or None to use CHANGES
     @type revision: varies depending on VC
 
@@ -288,16 +300,20 @@ class IBuildRequestStatus(Interface):
         IBuildStatus object) for each Build that is created to satisfy this
         request. There may be multiple Builds created in an attempt to handle
         the request: they may be interrupted by the user or abandoned due to
         a lost slave. The last Build (the one which actually gets to run to
         completion) is said to 'satisfy' the BuildRequest. The observer will
         be called once for each of these Builds, both old and new."""
     def unsubscribe(observer):
         """Unregister the callable that was registered with subscribe()."""
+    def getSubmitTime():
+        """Return the time when this request was submitted"""
+    def setSubmitTime(t):
+        """Sets the time when this request was submitted"""
 
 
 class ISlaveStatus(Interface):
     def getName():
         """Return the name of the build slave."""
 
     def getAdmin():
         """Return a string with the slave admin's contact data."""
@@ -439,17 +455,17 @@ class IEventSource(Interface):
 
 class IBuildStatus(Interface):
     """I represent the status of a single Build/BuildRequest. It could be
     in-progress or finished."""
 
     def getBuilder():
         """
         Return the BuilderStatus that owns this build.
-        
+
         @rtype: implementor of L{IBuilderStatus}
         """
 
     def isFinished():
         """Return a boolean. True means the build has finished, False means
         it is still running."""
 
     def waitUntilFinished():
@@ -533,21 +549,16 @@ class IBuildStatus(Interface):
         """Return the name of the buildslave which handled this build."""
 
     def getText():
         """Returns a list of strings to describe the build. These are
         intended to be displayed in a narrow column. If more space is
         available, the caller should join them together with spaces before
         presenting them to the user."""
 
-    def getColor():
-        """Returns a single string with the color that should be used to
-        display the build. 'green', 'orange', or 'red' are the most likely
-        ones."""
-
     def getResults():
         """Return a constant describing the results of the build: one of the
         constants in buildbot.status.builder: SUCCESS, WARNINGS, or
         FAILURE."""
 
     def getLogs():
         """Return a list of logs that describe the build as a whole. Some
         steps will contribute their logs, while others are are less important
@@ -658,21 +669,16 @@ class IBuildStepStatus(Interface):
     # Before ths step has finished, they all return None.
 
     def getText():
         """Returns a list of strings which describe the step. These are
         intended to be displayed in a narrow column. If more space is
         available, the caller should join them together with spaces before
         presenting them to the user."""
 
-    def getColor():
-        """Returns a single string with the color that should be used to
-        display this step. 'green', 'orange', 'red' and 'yellow' are the
-        most likely ones."""
-
     def getResults():
         """Return a tuple describing the results of the step: (result,
         strings). 'result' is one of the constants in
         buildbot.status.builder: SUCCESS, WARNINGS, FAILURE, or SKIPPED.
         'strings' is an optional list of strings that the step wants to
         append to the overall build's results. These strings are usually
         more terse than the ones returned by getText(): in particular,
         successful Steps do not usually contribute any text to the overall
@@ -701,20 +707,16 @@ class IStatusEvent(Interface):
         returned"""
 
     def getText():
         """Returns a list of strings which describe the event. These are
         intended to be displayed in a narrow column. If more space is
         available, the caller should join them together with spaces before
         presenting them to the user."""
 
-    def getColor():
-        """Returns a single string with the color that should be used to
-        display this event. 'red' and 'yellow' are the most likely ones."""
-
 
 LOG_CHANNEL_STDOUT = 0
 LOG_CHANNEL_STDERR = 1
 LOG_CHANNEL_HEADER = 2
 
 class IStatusLog(Interface):
     """I represent a single Log, which is a growing list of text items that
     contains some kind of output for a single BuildStep. I might be finished,
@@ -869,16 +871,22 @@ class IStatusReceiver(Interface):
     subscribed to an IStatus, an IBuilderStatus, or an IBuildStatus."""
 
     def buildsetSubmitted(buildset):
         """A new BuildSet has been submitted to the buildmaster.
 
         @type buildset: implementor of L{IBuildSetStatus}
         """
 
+    def requestSubmitted(request):
+        """A new BuildRequest has been submitted to the buildmaster.
+
+        @type request: implementor of L{IBuildRequestStatus}
+        """
+
     def builderAdded(builderName, builder):
         """
         A new Builder has just been added. This method may return an
         IStatusReceiver (probably 'self') which will be subscribed to receive
         builderChangedState and buildStarted/Finished events.
 
         @type  builderName: string
         @type  builder:     L{buildbot.status.builder.BuilderStatus}
@@ -918,16 +926,28 @@ class IStatusReceiver(Interface):
         invoked on the object for logs created by this one step. This
         receiver will be automatically unsubscribed when the step finishes.
 
         Alternatively, the method may return a tuple of an IStatusReceiver
         and an integer named 'updateInterval'. In addition to
         logStarted/logFinished messages, it will also receive stepETAUpdate
         messages about every updateInterval seconds."""
 
+    def stepTextChanged(build, step, text):
+        """The text for a step has been updated.
+
+        This is called when calling setText() on the step status, and
+        hands in the text list."""
+
+    def stepText2Changed(build, step, text2):
+        """The text2 for a step has been updated.
+
+        This is called when calling setText2() on the step status, and
+        hands in text2 list."""
+
     def stepETAUpdate(build, step, ETA, expectations):
         """This is a periodic update on the progress this Step has made
         towards completion. It gets an ETA (in seconds from the present) of
         when the step ought to be complete, and a list of expectation tuples
         (as returned by IBuildStepStatus.getExpectations) with more detailed
         information."""
 
     def logStarted(build, step, log):
@@ -1067,8 +1087,37 @@ class ILogObserver(Interface):
 
     # methods called by the LogFile
     def logChunk(build, step, log, channel, text):
         pass
 
 class IBuildSlave(Interface):
     # this is a marker interface for the BuildSlave class
     pass
+
+class ILatentBuildSlave(IBuildSlave):
+    """A build slave that is not always running, but can run when requested.
+    """
+    substantiated = Attribute('Substantiated',
+                              'Whether the latent build slave is currently '
+                              'substantiated with a real instance.')
+
+    def substantiate():
+        """Request that the slave substantiate with a real instance.
+
+        Returns a deferred that will callback when a real instance has
+        attached."""
+
+    # there is an insubstantiate too, but that is not used externally ATM.
+
+    def buildStarted(sb):
+        """Inform the latent build slave that a build has started.
+
+        ``sb`` is a LatentSlaveBuilder as defined in buildslave.py.  The sb
+        is the one for whom the build started.
+        """
+
+    def buildFinished(sb):
+        """Inform the latent build slave that a build has finished.
+
+        ``sb`` is a LatentSlaveBuilder as defined in buildslave.py.  The sb
+        is the one for whom the build finished.
+        """
--- a/buildbot/master.py
+++ b/buildbot/master.py
@@ -19,17 +19,17 @@ from twisted.persisted import styles
 
 import buildbot
 # sibling imports
 from buildbot.util import now
 from buildbot.pbutil import NewCredPerspective
 from buildbot.process.builder import Builder, IDLE
 from buildbot.process.base import BuildRequest
 from buildbot.status.builder import Status
-from buildbot.changes.changes import Change, ChangeMaster
+from buildbot.changes.changes import Change, ChangeMaster, TestChangeMaster
 from buildbot.sourcestamp import SourceStamp
 from buildbot.buildslave import BuildSlave
 from buildbot import interfaces, locks
 from buildbot.process.properties import Properties
 
 ########################################
 
 class BotMaster(service.MultiService):
@@ -57,16 +57,20 @@ class BotMaster(service.MultiService):
         # connected, that attribute will hold None.
         self.slaves = {} # maps slavename to BuildSlave
         self.statusClientService = None
         self.watchers = {}
 
         # self.locks holds the real Lock instances
         self.locks = {}
 
+        # self.mergeRequests is the callable override for merging build
+        # requests
+        self.mergeRequests = None
+
     # these four are convenience functions for testing
 
     def waitUntilBuilderAttached(self, name):
         b = self.builders[name]
         #if b.slaves:
         #    return defer.succeed(None)
         d = defer.Deferred()
         b.watchers['attach'].append(d)
@@ -184,19 +188,40 @@ class BotMaster(service.MultiService):
         return d
 
     def _updateAllSlaves(self):
         """Notify all buildslaves about changes in their Builders."""
         dl = [s.updateSlave() for s in self.slaves.values()]
         return defer.DeferredList(dl)
 
     def maybeStartAllBuilds(self):
-        for b in self.builders.values():
+        builders = self.builders.values()
+        def _sortfunc(b1, b2):
+            t1 = b1.getOldestRequestTime()
+            t2 = b2.getOldestRequestTime()
+            # If t1 or t2 is None, then there are no build requests,
+            # so sort it at the end
+            if t1 is None:
+                return 1
+            if t2 is None:
+                return -1
+            return cmp(t1, t2)
+        builders.sort(cmp=_sortfunc)
+        for b in builders:
             b.maybeStartBuild()
 
+    def shouldMergeRequests(self, builder, req1, req2):
+        """Determine whether two BuildRequests should be merged for
+        the given builder.
+
+        """
+        if self.mergeRequests is not None:
+            return self.mergeRequests(builder, req1, req2)
+        return req1.canBeMergedWith(req2)
+
     def getPerspective(self, slavename):
         return self.slaves[slavename]
 
     def shutdownSlaves(self):
         # TODO: make this into a bot method rather than a builder method
         for b in self.slaves.values():
             b.shutdownSlave()
 
@@ -375,17 +400,17 @@ class BuildMaster(service.MultiService, 
 
         self.status = Status(self.botmaster, self.basedir)
 
         self.statusTargets = []
 
         # this ChangeMaster is a dummy, only used by tests. In the real
         # buildmaster, where the BuildMaster instance is activated
         # (startService is called) by twistd, this attribute is overwritten.
-        self.useChanges(ChangeMaster())
+        self.useChanges(TestChangeMaster())
 
         self.readConfig = False
 
     def upgradeToVersion1(self):
         self.dispatcher = self.slaveFactory.root.portal.realm
 
     def upgradeToVersion2(self): # post-0.4.3
         self.webServer = self.webTCPPort
@@ -493,20 +518,20 @@ class BuildMaster(service.MultiService, 
             config = localDict['BuildmasterConfig']
         except KeyError:
             log.err("missing config dictionary")
             log.err("config file must define BuildmasterConfig")
             raise
 
         known_keys = ("bots", "slaves",
                       "sources", "change_source",
-                      "schedulers", "builders",
-                      "slavePortnum", "debugPassword", "manhole",
-                      "status", "projectName", "projectURL", "buildbotURL",
-                      "properties"
+                      "schedulers", "builders", "mergeRequests", 
+                      "slavePortnum", "debugPassword", "logCompressionLimit",
+                      "manhole", "status", "projectName", "projectURL",
+                      "buildbotURL", "properties"
                       )
         for k in config.keys():
             if k not in known_keys:
                 log.msg("unknown key '%s' defined in config dictionary" % k)
 
         try:
             # required
             schedulers = config['schedulers']
@@ -525,16 +550,23 @@ class BuildMaster(service.MultiService, 
             # optional
             debugPassword = config.get('debugPassword')
             manhole = config.get('manhole')
             status = config.get('status', [])
             projectName = config.get('projectName')
             projectURL = config.get('projectURL')
             buildbotURL = config.get('buildbotURL')
             properties = config.get('properties', {})
+            logCompressionLimit = config.get('logCompressionLimit')
+            if logCompressionLimit is not None and not \
+                    isinstance(logCompressionLimit, int):
+                raise ValueError("logCompressionLimit needs to be bool or int")
+            mergeRequests = config.get('mergeRequests')
+            if mergeRequests is not None and not callable(mergeRequests):
+                raise ValueError("mergeRequests must be a callable")
 
         except KeyError, e:
             log.msg("config dictionary is missing a required parameter")
             log.msg("leaving old configuration in place")
             raise
 
         #if "bots" in config:
         #    raise KeyError("c['bots'] is no longer accepted")
@@ -566,19 +598,20 @@ class BuildMaster(service.MultiService, 
                  "removed by 0.8.0 . Please use c['change_source'] instead.")
             log.msg(m)
             warnings.warn(m, DeprecationWarning)
             for s in config['sources']:
                 change_sources.append(s)
 
         # do some validation first
         for s in slaves:
-            assert isinstance(s, BuildSlave)
+            assert interfaces.IBuildSlave.providedBy(s)
             if s.slavename in ("debug", "change", "status"):
-                raise KeyError, "reserved name '%s' used for a bot" % s.slavename
+                raise KeyError(
+                    "reserved name '%s' used for a bot" % s.slavename)
         if config.has_key('interlocks'):
             raise KeyError("c['interlocks'] is no longer accepted")
 
         assert isinstance(change_sources, (list, tuple))
         for s in change_sources:
             assert interfaces.IChangeSource(s, None)
         # this assertion catches c['schedulers'] = Scheduler(), since
         # Schedulers are service.MultiServices and thus iterable.
@@ -677,19 +710,23 @@ class BuildMaster(service.MultiService, 
         # TODO: actually, this is spread across a couple of Deferreds, so it
         # really isn't atomic.
 
         d = defer.succeed(None)
 
         self.projectName = projectName
         self.projectURL = projectURL
         self.buildbotURL = buildbotURL
-        
+
         self.properties = Properties()
         self.properties.update(properties, self.configFileName)
+        if logCompressionLimit is not None:
+            self.status.logCompressionLimit = logCompressionLimit
+        if mergeRequests is not None:
+            self.botmaster.mergeRequests = mergeRequests
 
         # self.slaves: Disconnect any that were attached and removed from the
         # list. Update self.checker with the new list of passwords, including
         # debug/change/status.
         d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 
         # self.debugPassword
         if debugPassword:
@@ -776,20 +813,31 @@ class BuildMaster(service.MultiService, 
 
 
     def loadConfig_Schedulers(self, newschedulers):
         oldschedulers = self.allSchedulers()
         removed = [s for s in oldschedulers if s not in newschedulers]
         added = [s for s in newschedulers if s not in oldschedulers]
         dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed]
         def addNewOnes(res):
+            log.msg("adding %d new schedulers, removed %d" % 
+                    (len(added), len(dl)))
             for s in added:
                 s.setServiceParent(self)
         d = defer.DeferredList(dl, fireOnOneErrback=1)
         d.addCallback(addNewOnes)
+        if removed or added:
+            # notify Downstream schedulers to potentially pick up
+            # new schedulers now that we have removed and added some
+            def updateDownstreams(res):
+                log.msg("notifying downstream schedulers of changes")
+                for s in newschedulers:
+                    if interfaces.IDownstreamScheduler.providedBy(s):
+                        s.checkUpstreamScheduler()
+            d.addCallback(updateDownstreams)
         return d
 
     def loadConfig_Builders(self, newBuilderData):
         somethingChanged = False
         newList = {}
         newBuilderNames = []
         allBuilders = self.botmaster.builders.copy()
         for data in newBuilderData:
@@ -910,9 +958,8 @@ class Control:
     def getBuilder(self, name):
         b = self.master.botmaster.builders[name]
         return interfaces.IBuilderControl(b)
 
 components.registerAdapter(Control, BuildMaster, interfaces.IControl)
 
 # so anybody who can get a handle on the BuildMaster can cause a build with:
 #  IControl(master).getBuilder("full-2.3").requestBuild(buildrequest)
-
--- a/buildbot/process/base.py
+++ b/buildbot/process/base.py
@@ -37,33 +37,36 @@ class BuildRequest:
 
     @type reason: string
     @ivar reason: the reason this Build is being requested. Schedulers
                   provide this, but for forced builds the user requesting the
                   build will provide a string.
 
     @type properties: Properties object
     @ivar properties: properties that should be applied to this build
+                      'owner' property is used by Build objects to collect
+                      the list returned by getInterestedUsers
 
     @ivar status: the IBuildStatus object which tracks our status
 
     @ivar submittedAt: a timestamp (seconds since epoch) when this request
                        was submitted to the Builder. This is used by the CVS
-                       step to compute a checkout timestamp.
+                       step to compute a checkout timestamp, as well as the
+                       master to prioritize build requests from oldest to
+                       newest.
     """
 
     source = None
     builder = None
     startCount = 0 # how many times we have tried to start this build
+    submittedAt = None
 
     implements(interfaces.IBuildRequestControl)
 
-    def __init__(self, reason, source, builderName=None, properties=None):
-        # TODO: remove the =None on builderName, it is there so I don't have
-        # to change a lot of tests that create BuildRequest objects
+    def __init__(self, reason, source, builderName, properties=None):
         assert interfaces.ISourceStamp(source, None)
         self.reason = reason
         self.source = source
 
         self.properties = Properties()
         if properties:
             self.properties.updateFromProperties(properties)
 
@@ -132,16 +135,23 @@ class BuildRequest:
         """Cancel this request. This can only be successful if the Build has
         not yet been started.
 
         @return: a boolean indicating if the cancel was successful."""
         if self.builder:
             return self.builder.cancelBuildRequest(self)
         return False
 
+    def setSubmitTime(self, t):
+        self.submittedAt = t
+        self.status.setSubmitTime(t)
+
+    def getSubmitTime(self):
+        return self.submittedAt
+
 
 class Build:
     """I represent a single build by a single slave. Specialized Builders can
     use subclasses of Build to hold status information unique to those build
     processes.
 
     I control B{how} the build proceeds. The actual build is broken up into a
     series of steps, saved in the .buildSteps[] array as a list of
@@ -177,27 +187,32 @@ class Build:
         # build a source stamp
         self.source = requests[0].mergeWith(requests[1:])
         self.reason = requests[0].mergeReasons(requests[1:])
 
         self.progress = None
         self.currentStep = None
         self.slaveEnvironment = {}
 
+        self.terminate = False
+
     def setBuilder(self, builder):
         """
         Set the given builder as our builder.
 
         @type  builder: L{buildbot.process.builder.Builder}
         """
         self.builder = builder
 
     def setLocks(self, locks):
         self.locks = locks
 
+    def setSlaveEnvironment(self, env):
+        self.slaveEnvironment = env
+
     def getSourceStamp(self):
         return self.source
 
     def setProperty(self, propname, value, source):
         """Set a property on this build. This may only be called after the
         build has started, so that it has a BuildStatus object where the
         properties can live."""
         self.build_status.setProperty(propname, value, source)
@@ -338,18 +353,17 @@ class Build:
             # the build hasn't started yet, so log the exception as a point
             # event instead of flunking the build. TODO: associate this
             # failure with the build instead. this involves doing
             # self.build_status.buildStarted() from within the exception
             # handler
             log.msg("Build.setupBuild failed")
             log.err(Failure())
             self.builder.builder_status.addPointEvent(["setupBuild",
-                                                       "exception"],
-                                                      color="purple")
+                                                       "exception"])
             self.finished = True
             self.results = FAILURE
             self.deferred = None
             d.callback(self)
             return d
 
         self.acquireLocks().addCallback(self._startBuild_2)
         return d
@@ -389,20 +403,23 @@ class Build:
                 log.msg("error while creating step, factory=%s, args=%s"
                         % (factory, args))
                 raise
             step.setBuild(self)
             step.setBuildSlave(self.slavebuilder.slave)
             step.setDefaultWorkdir(self.workdir)
             name = step.name
             count = 1
-            while name in stepnames and count < 100:
+            while name in stepnames and count < 1000:
                 count += 1
                 name = step.name + "_%d" % count
-            if name in stepnames:
+            if count == 1000:
+                raise RuntimeError("reached 1000 steps with base name" + \
+                                   "%s, bailing" % step.name)
+            elif name in stepnames:
                 raise RuntimeError("duplicate step '%s'" % step.name)
             step.name = name
             stepnames.append(name)
             self.steps.append(step)
 
             # tell the BuildStatus about the step. This will create a
             # BuildStepStatus and bind it to the Step.
             step_status = self.build_status.addStepWithName(name)
@@ -426,31 +443,45 @@ class Build:
 
         if self.useProgress:
             self.progress = BuildProgress(sps)
             if self.progress and expectations:
                 self.progress.setExpectationsFrom(expectations)
 
         # we are now ready to set up our BuildStatus.
         self.build_status.setSourceStamp(self.source)
+        self.build_status.setRequests([req.status for req in self.requests])
         self.build_status.setReason(self.reason)
         self.build_status.setBlamelist(self.blamelist())
         self.build_status.setProgress(self.progress)
 
+        # gather owners from build requests
+        owners = [r.properties['owner'] for r in self.requests
+                  if r.properties.has_key('owner')]
+        if owners: self.setProperty('owners', owners, self.reason)
+
         self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED
         self.result = SUCCESS # overall result, may downgrade after each step
         self.text = [] # list of text string lists (text2)
 
     def getNextStep(self):
         """This method is called to obtain the next BuildStep for this build.
         When it returns None (or raises a StopIteration exception), the build
         is complete."""
         if not self.steps:
             return None
-        return self.steps.pop(0)
+        if self.terminate:
+            while True:
+                s = self.steps.pop(0)
+                if s.alwaysRun:
+                    return s
+                if not self.steps:
+                    return None
+        else:
+            return self.steps.pop(0)
 
     def startNextStep(self):
         try:
             s = self.getNextStep()
         except StopIteration:
             s = None
         if not s:
             return self.allStepsDone()
@@ -460,18 +491,18 @@ class Build:
         d.addErrback(self.buildException)
 
     def _stepDone(self, results, step):
         self.currentStep = None
         if self.finished:
             return # build was interrupted, don't keep building
         terminate = self.stepDone(results, step) # interpret/merge results
         if terminate:
-            return self.allStepsDone()
-        self.startNextStep()
+            self.terminate = True
+        return self.startNextStep()
 
     def stepDone(self, result, step):
         """This method is called when the BuildStep completes. It is passed a
         status object from the BuildStep and is responsible for merging the
         Step's results into those of the overall Build."""
 
         terminate = False
         text = None
@@ -486,17 +517,16 @@ class Build:
             terminate = True
         if result == FAILURE:
             if step.warnOnFailure:
                 if self.result != FAILURE:
                     self.result = WARNINGS
             if step.flunkOnFailure:
                 self.result = FAILURE
             if step.haltOnFailure:
-                self.result = FAILURE
                 terminate = True
         elif result == WARNINGS:
             if step.warnOnWarnings:
                 if self.result != FAILURE:
                     self.result = WARNINGS
             if step.flunkOnWarnings:
                 self.result = FAILURE
         elif result == EXCEPTION:
@@ -530,63 +560,57 @@ class Build:
         self.builder.builder_status.addPointEvent(['interrupt'])
         self.currentStep.interrupt(reason)
         if 0:
             # TODO: maybe let its deferred do buildFinished
             if self.currentStep and self.currentStep.progress:
                 # XXX: really .fail or something
                 self.currentStep.progress.finish()
             text = ["stopped", reason]
-            self.buildFinished(text, "red", FAILURE)
+            self.buildFinished(text, FAILURE)
 
     def allStepsDone(self):
         if self.result == FAILURE:
-            color = "red"
             text = ["failed"]
         elif self.result == WARNINGS:
-            color = "orange"
             text = ["warnings"]
         elif self.result == EXCEPTION:
-            color = "purple"
             text = ["exception"]
         else:
-            color = "green"
             text = ["build", "successful"]
         text.extend(self.text)
-        return self.buildFinished(text, color, self.result)
+        return self.buildFinished(text, self.result)
 
     def buildException(self, why):
         log.msg("%s.buildException" % self)
         log.err(why)
-        self.buildFinished(["build", "exception"], "purple", FAILURE)
+        self.buildFinished(["build", "exception"], FAILURE)
 
-    def buildFinished(self, text, color, results):
+    def buildFinished(self, text, results):
         """This method must be called when the last Step has completed. It
         marks the Build as complete and returns the Builder to the 'idle'
         state.
 
-        It takes three arguments which describe the overall build status:
-        text, color, results. 'results' is one of SUCCESS, WARNINGS, or
-        FAILURE.
+        It takes two arguments which describe the overall build status:
+        text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE.
 
         If 'results' is SUCCESS or WARNINGS, we will permit any dependant
         builds to start. If it is 'FAILURE', those builds will be
         abandoned."""
 
         self.finished = True
         if self.remote:
             self.remote.dontNotifyOnDisconnect(self.lostRemote)
         self.results = results
 
         log.msg(" %s: build finished" % self)
         self.build_status.setText(text)
-        self.build_status.setColor(color)
         self.build_status.setResults(results)
         self.build_status.buildFinished()
-        if self.progress:
+        if self.progress and results == SUCCESS:
             # XXX: also test a 'timing consistent' flag?
             log.msg(" setting expectations for next time")
             self.builder.setExpectations(self.progress)
         reactor.callLater(0, self.releaseLocks)
         self.deferred.callback(self)
         self.deferred = None
 
     def releaseLocks(self):
--- a/buildbot/process/builder.py
+++ b/buildbot/process/builder.py
@@ -9,39 +9,41 @@ from buildbot import interfaces
 from buildbot.status.progress import Expectations
 from buildbot.util import now
 from buildbot.process import base
 
 (ATTACHING, # slave attached, still checking hostinfo/etc
  IDLE, # idle, available for use
  PINGING, # build about to start, making sure it is still alive
  BUILDING, # build is running
- ) = range(4)
+ LATENT, # latent slave is not substantiated; similar to idle
+ ) = range(5)
 
-class SlaveBuilder(pb.Referenceable):
+
+class AbstractSlaveBuilder(pb.Referenceable):
     """I am the master-side representative for one of the
     L{buildbot.slave.bot.SlaveBuilder} objects that lives in a remote
     buildbot. When a remote builder connects, I query it for command versions
     and then make it available to any Builds that are ready to run. """
 
     def __init__(self):
         self.ping_watchers = []
-        self.state = ATTACHING
+        self.state = None # set in subclass
         self.remote = None
         self.slave = None
         self.builder_name = None
 
     def __repr__(self):
-        r = "<SlaveBuilder"
+        r = ["<", self.__class__.__name__]
         if self.builder_name:
-            r += " builder=%s" % self.builder_name
+            r.extend([" builder=", self.builder_name])
         if self.slave:
-            r += " slave=%s" % self.slave.slavename
-        r += ">"
-        return r
+            r.extend([" slave=", self.slave.slavename])
+        r.append(">")
+        return ''.join(r)
 
     def setBuilder(self, b):
         self.builder = b
         self.builder_name = b.name
 
     def getSlaveCommandVersion(self, command, oldversion=None):
         if self.remoteCommands is None:
             # the slave is 0.5.0 or earlier
@@ -56,32 +58,43 @@ class SlaveBuilder(pb.Referenceable):
         # otherwise, check in with the BuildSlave
         if self.slave:
             return self.slave.canStartBuild()
 
         # no slave? not very available.
         return False
 
     def isBusy(self):
-        return self.state != IDLE
+        return self.state not in (IDLE, LATENT)
+
+    def buildStarted(self):
+        self.state = BUILDING
+
+    def buildFinished(self):
+        self.state = IDLE
+        reactor.callLater(0, self.builder.botmaster.maybeStartAllBuilds)
 
     def attached(self, slave, remote, commands):
         """
         @type  slave: L{buildbot.buildslave.BuildSlave}
         @param slave: the BuildSlave that represents the buildslave as a
                       whole
         @type  remote: L{twisted.spread.pb.RemoteReference}
         @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder}
         @type  commands: dict: string -> string, or None
         @param commands: provides the slave's version of each RemoteCommand
         """
-        self.slave = slave
+        self.state = ATTACHING
         self.remote = remote
         self.remoteCommands = commands # maps command name to version
-        self.slave.addSlaveBuilder(self)
+        if self.slave is None:
+            self.slave = slave
+            self.slave.addSlaveBuilder(self)
+        else:
+            assert self.slave == slave
         log.msg("Buildslave %s attached to %s" % (slave.slavename,
                                                   self.builder_name))
         d = self.remote.callRemote("setMaster", self)
         d.addErrback(self._attachFailure, "Builder.setMaster")
         d.addCallback(self._attached2)
         return d
 
     def _attached2(self, res):
@@ -96,70 +109,67 @@ class SlaveBuilder(pb.Referenceable):
         return self
 
     def _attachFailure(self, why, where):
         assert isinstance(where, str)
         log.msg(where)
         log.err(why)
         return why
 
-    def detached(self):
-        log.msg("Buildslave %s detached from %s" % (self.slave.slavename,
-                                                    self.builder_name))
-        if self.slave:
-            self.slave.removeSlaveBuilder(self)
-        self.slave = None
-        self.remote = None
-        self.remoteCommands = None
-
-    def buildStarted(self):
-        self.state = BUILDING
-
-    def buildFinished(self):
-        self.state = IDLE
-        reactor.callLater(0, self.builder.botmaster.maybeStartAllBuilds)
-
     def ping(self, timeout, status=None):
         """Ping the slave to make sure it is still there. Returns a Deferred
         that fires with True if it is.
 
         @param status: if you point this at a BuilderStatus, a 'pinging'
                        event will be pushed.
         """
-
+        oldstate = self.state
         self.state = PINGING
         newping = not self.ping_watchers
         d = defer.Deferred()
         self.ping_watchers.append(d)
         if newping:
             if status:
-                event = status.addEvent(["pinging"], "yellow")
+                event = status.addEvent(["pinging"])
                 d2 = defer.Deferred()
                 d2.addCallback(self._pong_status, event)
                 self.ping_watchers.insert(0, d2)
                 # I think it will make the tests run smoother if the status
                 # is updated before the ping completes
             Ping().ping(self.remote, timeout).addCallback(self._pong)
 
+        def reset_state(res):
+            if self.state == PINGING:
+                self.state = oldstate
+            return res
+        d.addCallback(reset_state)
         return d
 
     def _pong(self, res):
         watchers, self.ping_watchers = self.ping_watchers, []
         for d in watchers:
             d.callback(res)
 
     def _pong_status(self, res, event):
         if res:
             event.text = ["ping", "success"]
-            event.color = "green"
         else:
             event.text = ["ping", "failed"]
-            event.color = "red"
         event.finish()
 
+    def detached(self):
+        log.msg("Buildslave %s detached from %s" % (self.slave.slavename,
+                                                    self.builder_name))
+        if self.slave:
+            self.slave.removeSlaveBuilder(self)
+        self.slave = None
+        self.remote = None
+        self.remoteCommands = None
+
+
 class Ping:
     running = False
     timer = None
 
     def ping(self, remote, timeout):
         assert not self.running
         self.running = True
         log.msg("sending ping")
@@ -208,16 +218,97 @@ class Ping:
         # creating a nasty loop.
         remote.broker.transport.loseConnection()
         # TODO: except, if they actually did manage to get this far, they'll
         # probably reconnect right away, and we'll do this game again. Maybe
         # it would be better to leave them in the PINGING state.
         self.d.callback(False)
 
 
+class SlaveBuilder(AbstractSlaveBuilder):
+
+    def __init__(self):
+        AbstractSlaveBuilder.__init__(self)
+        self.state = ATTACHING
+
+    def detached(self):
+        AbstractSlaveBuilder.detached(self)
+        if self.slave:
+            self.slave.removeSlaveBuilder(self)
+        self.slave = None
+        self.state = ATTACHING
+
+    def buildFinished(self):
+        # Call the slave's buildFinished if we can; the slave may be waiting
+        # to do a graceful shutdown and needs to know when it's idle.
+        # After, we check to see if we can start other builds.
+        self.state = IDLE
+        if self.slave:
+            d = self.slave.buildFinished(self)
+            d.addCallback(lambda x: reactor.callLater(0, self.builder.botmaster.maybeStartAllBuilds))
+        else:
+            reactor.callLater(0, self.builder.botmaster.maybeStartAllBuilds)
+
+
+class LatentSlaveBuilder(AbstractSlaveBuilder):
+    def __init__(self, slave, builder):
+        AbstractSlaveBuilder.__init__(self)
+        self.slave = slave
+        self.state = LATENT
+        self.setBuilder(builder)
+        self.slave.addSlaveBuilder(self)
+        log.msg("Latent buildslave %s attached to %s" % (slave.slavename,
+                                                         self.builder_name))
+
+    def substantiate(self, build):
+        d = self.slave.substantiate(self)
+        if not self.slave.substantiated:
+            event = self.builder.builder_status.addEvent(
+                ["substantiating"])
+            def substantiated(res):
+                msg = ["substantiate", "success"]
+                if isinstance(res, basestring):
+                    msg.append(res)
+                elif isinstance(res, (tuple, list)):
+                    msg.extend(res)
+                event.text = msg
+                event.finish()
+                return res
+            def substantiation_failed(res):
+                event.text = ["substantiate", "failed"]
+                # TODO add log of traceback to event
+                event.finish()
+                return res
+            d.addCallbacks(substantiated, substantiation_failed)
+        return d
+
+    def detached(self):
+        AbstractSlaveBuilder.detached(self)
+        self.state = LATENT
+
+    def buildStarted(self):
+        AbstractSlaveBuilder.buildStarted(self)
+        self.slave.buildStarted(self)
+
+    def buildFinished(self):
+        AbstractSlaveBuilder.buildFinished(self)
+        self.slave.buildFinished(self)
+
+    def _attachFailure(self, why, where):
+        self.state = LATENT
+        return AbstractSlaveBuilder._attachFailure(self, why, where)
+
+    def ping(self, timeout, status=None):
+        if not self.slave.substantiated:
+            if status:
+                status.addEvent(["ping", "latent"]).finish()
+            return defer.succeed(True)
+        return AbstractSlaveBuilder.ping(self, timeout, status)
+
+
 class Builder(pb.Referenceable):
     """I manage all Builds of a given type.
 
     Each Builder is created by an entry in the config file (the c['builders']
     list), with a number of parameters.
 
     One of these parameters is the L{buildbot.process.factory.BuildFactory}
     object that is associated with this Builder. The factory is responsible
@@ -273,16 +364,18 @@ class Builder(pb.Referenceable):
         self.slavenames = []
         if setup.has_key('slavename'):
             self.slavenames.append(setup['slavename'])
         if setup.has_key('slavenames'):
             self.slavenames.extend(setup['slavenames'])
         self.builddir = setup['builddir']
         self.buildFactory = setup['factory']
         self.locks = setup.get("locks", [])
+        self.env = setup.get('env', {})
+        assert isinstance(self.env, dict)
         if setup.has_key('periodicBuildTime'):
             raise ValueError("periodicBuildTime can no longer be defined as"
                              " part of the Builder: use scheduler.Periodic"
                              " instead")
 
         # build/wannabuild slots: Build objects move along this sequence
         self.buildable = []
         self.building = []
@@ -318,29 +411,37 @@ class Builder(pb.Referenceable):
             diffs.append('slavenames changed from %s to %s' \
                          % (self.slavenames, setup_slavenames))
         if setup['builddir'] != self.builddir:
             diffs.append('builddir changed from %s to %s' \
                          % (self.builddir, setup['builddir']))
         if setup['factory'] != self.buildFactory: # compare objects
             diffs.append('factory changed')
         oldlocks = [(lock.__class__, lock.name)
-                    for lock in setup.get('locks',[])]
+                    for lock in self.locks]
         newlocks = [(lock.__class__, lock.name)
-                    for lock in self.locks]
+                    for lock in setup.get('locks',[])]
         if oldlocks != newlocks:
             diffs.append('locks changed from %s to %s' % (oldlocks, newlocks))
         return diffs
 
     def __repr__(self):
         return "<Builder '%s' at %d>" % (self.name, id(self))
 
+    def getOldestRequestTime(self):
+        """Returns the timestamp of the oldest build request for this builder.
+
+        If there are no build requests, None is returned."""
+        if self.buildable:
+            return self.buildable[0].getSubmitTime()
+        else:
+            return None
 
     def submitBuildRequest(self, req):
-        req.submittedAt = now()
+        req.setSubmitTime(now())
         self.buildable.append(req)
         req.requestSubmitted(self)
         self.builder_status.addBuildRequest(req.status)
         self.maybeStartBuild()
 
     def cancelBuildRequest(self, req):
         if req in self.buildable:
             self.buildable.remove(req)
@@ -438,16 +539,28 @@ class Builder(pb.Referenceable):
     def fireTestEvent(self, name, fire_with=None):
         if fire_with is None:
             fire_with = self
         watchers = self.watchers[name]
         self.watchers[name] = []
         for w in watchers:
             reactor.callLater(0, w.callback, fire_with)
 
+    def addLatentSlave(self, slave):
+        assert interfaces.ILatentBuildSlave.providedBy(slave)
+        for s in self.slaves:
+            if s == slave:
+                break
+        else:
+            sb = LatentSlaveBuilder(slave, self)
+            self.builder_status.addPointEvent(
+                ['added', 'latent', slave.slavename])
+            self.slaves.append(sb)
+            reactor.callLater(0, self.maybeStartBuild)
+
     def attached(self, slave, remote, commands):
         """This is invoked by the BuildSlave when the self.slavename bot
         registers their builder.
 
         @type  slave: L{buildbot.buildslave.BuildSlave}
         @param slave: the BuildSlave that represents the buildslave as a whole
         @type  remote: L{twisted.spread.pb.RemoteReference}
         @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder}
@@ -560,62 +673,79 @@ class Builder(pb.Referenceable):
         # pick an idle slave
         available_slaves = [sb for sb in self.slaves if sb.isAvailable()]
         if not available_slaves:
             log.msg("%s: want to start build, but we don't have a remote"
                     % self)
             self.updateBigStatus()
             return
         if self.CHOOSE_SLAVES_RANDOMLY:
+            # TODO prefer idle over latent? maybe other sorting preferences?
             sb = random.choice(available_slaves)
         else:
             sb = available_slaves[0]
 
         # there is something to build, and there is a slave on which to build
         # it. Grab the oldest request, see if we can merge it with anything
         # else.
         req = self.buildable.pop(0)
         self.builder_status.removeBuildRequest(req.status)
         mergers = []
+        botmaster = self.botmaster
         for br in self.buildable[:]:
-            if req.canBeMergedWith(br):
+            if botmaster.shouldMergeRequests(self, req, br):
                 self.buildable.remove(br)
                 self.builder_status.removeBuildRequest(br.status)
                 mergers.append(br)
         requests = [req] + mergers
 
         # Create a new build from our build factory and set ourself as the
         # builder.
         build = self.buildFactory.newBuild(requests)
         build.setBuilder(self)
         build.setLocks(self.locks)
+        if len(self.env) > 0:
+            build.setSlaveEnvironment(self.env)
 
         # start it
         self.startBuild(build, sb)
 
     def startBuild(self, build, sb):
         """Start a build on the given slave.
         @param build: the L{base.Build} to start
         @param sb: the L{SlaveBuilder} which will host this build
 
         @return: a Deferred which fires with a
         L{buildbot.interfaces.IBuildControl} that can be used to stop the
         Build, or to access a L{buildbot.interfaces.IBuildStatus} which will
         watch the Build as it runs. """
 
         self.building.append(build)
         self.updateBigStatus()
-
-        log.msg("starting build %s.. pinging the slave %s" % (build, sb))
+        if isinstance(sb, LatentSlaveBuilder):
+            log.msg("starting build %s.. substantiating the slave %s" %
+                    (build, sb))
+            d = sb.substantiate(build)
+            def substantiated(res):
+                return sb.ping(self.START_BUILD_TIMEOUT)
+            def substantiation_failed(res):
+                self.builder_status.addPointEvent(
+                    ['removing', 'latent', sb.slave.slavename])
+                sb.slave.disconnect()
+                # TODO: should failover to a new Build
+                #self.retryBuild(sb.build)
+            d.addCallbacks(substantiated, substantiation_failed)
+        else:
+            log.msg("starting build %s.. pinging the slave %s" % (build, sb))
+            d = sb.ping(self.START_BUILD_TIMEOUT)
         # ping the slave to make sure they're still there. If they're fallen
         # off the map (due to a NAT timeout or something), this will fail in
         # a couple of minutes, depending upon the TCP timeout. TODO: consider
         # making this time out faster, or at least characterize the likely
         # duration.
-        d = sb.ping(self.START_BUILD_TIMEOUT)
         d.addCallback(self._startBuild_1, build, sb)
         return d
 
     def _startBuild_1(self, res, build, sb):
         if not res:
             return self._startBuildFailed("slave ping failed", build, sb)
         # The buildslave is ready to go. sb.buildStarted() sets its state to
         # BUILDING (so we won't try to use it for any other builds). This
@@ -721,18 +851,17 @@ class BuilderControl(components.Adapter)
         # return IBuildRequestControl objects
         raise NotImplementedError
 
     def getBuild(self, number):
         return self.original.getBuild(number)
 
     def ping(self, timeout=30):
         if not self.original.slaves:
-            self.original.builder_status.addPointEvent(["ping", "no slave"],
-                                                       "red")
+            self.original.builder_status.addPointEvent(["ping", "no slave"])
             return defer.succeed(False) # interfaces.NoSlaveError
         dl = []
         for s in self.original.slaves:
             dl.append(s.ping(timeout, self.original.builder_status))
         d = defer.DeferredList(dl)
         d.addCallback(self._gatherPingResults)
         return d
 
--- a/buildbot/process/buildstep.py
+++ b/buildbot/process/buildstep.py
@@ -428,17 +428,17 @@ class LogLineObserver(LogObserver):
 class RemoteShellCommand(LoggedRemoteCommand):
     """This class helps you run a shell command on the build slave. It will
     accumulate all the command's output into a Log named 'stdio'. When the
     command is finished, it will fire a Deferred. You can then check the
     results of the command and parse the output however you like."""
 
     def __init__(self, workdir, command, env=None, 
                  want_stdout=1, want_stderr=1,
-                 timeout=20*60, logfiles={}, **kwargs):
+                 timeout=20*60, logfiles={}, usePTY="slave-config"):
         """
         @type  workdir: string
         @param workdir: directory where the command ought to run,
                         relative to the Builder's home directory. Defaults to
                         '.': the same as the Builder's homedir. This should
                         probably be '.' for the initial 'cvs checkout'
                         command (which creates a workdir), and the Build-wide
                         workdir for all subsequent commands (including
@@ -481,33 +481,34 @@ class RemoteShellCommand(LoggedRemoteCom
             # able to modify the original.
             env = env.copy()
         args = {'workdir': workdir,
                 'env': env,
                 'want_stdout': want_stdout,
                 'want_stderr': want_stderr,
                 'logfiles': logfiles,
                 'timeout': timeout,
+                'usePTY': usePTY,
                 }
         LoggedRemoteCommand.__init__(self, "shell", args)
 
     def start(self):
         self.args['command'] = self.command
         if self.remote_command == "shell":
             # non-ShellCommand slavecommands are responsible for doing this
             # fixup themselves
             if self.step.slaveVersion("shell", "old") == "old":
                 self.args['dir'] = self.args['workdir']
         what = "command '%s' in dir '%s'" % (self.args['command'],
                                              self.args['workdir'])
         log.msg(what)
         return LoggedRemoteCommand.start(self)
 
     def __repr__(self):
-        return "<RemoteShellCommand '%s'>" % self.command
+        return "<RemoteShellCommand '%s'>" % repr(self.command)
 
 class BuildStep:
     """
     I represent a single step of the build process. This step may involve
     zero or more commands to be run in the build slave, as well as arbitrary
     processing on the master side. Regardless of how many slave commands are
     run, the BuildStep will result in a single status value.
 
@@ -539,35 +540,39 @@ class BuildStep:
     @ivar step_status: collects output status
     """
 
     # these parameters are used by the parent Build object to decide how to
     # interpret our results. haltOnFailure will affect the build process
     # immediately, the others will be taken into consideration when
     # determining the overall build status.
     #
+    # steps that are makred as alwaysRun will be run regardless of the outcome
+    # of previous steps (especially steps with haltOnFailure=True)
     haltOnFailure = False
     flunkOnWarnings = False
     flunkOnFailure = False
     warnOnWarnings = False
     warnOnFailure = False
+    alwaysRun = False
 
     # 'parms' holds a list of all the parameters we care about, to allow
     # users to instantiate a subclass of BuildStep with a mixture of
     # arguments, some of which are for us, some of which are for the subclass
     # (or a delegate of the subclass, like how ShellCommand delivers many
     # arguments to the RemoteShellCommand that it creates). Such delegating
     # subclasses will use this list to figure out which arguments are meant
     # for us and which should be given to someone else.
     parms = ['name', 'locks',
              'haltOnFailure',
              'flunkOnWarnings',
              'flunkOnFailure',
              'warnOnWarnings',
              'warnOnFailure',
+             'alwaysRun',
              'progressMetrics',
              ]
 
     name = "generic"
     locks = []
     progressMetrics = () # 'time' is implicit
     useProgress = True # set to False if step is really unpredictable
     build = None
@@ -593,21 +598,21 @@ class BuildStep:
         # available during __init__, but setBuild() will be called just
         # afterwards.
         self.build = build
 
     def setBuildSlave(self, buildslave):
         self.buildslave = buildslave
 
     def setDefaultWorkdir(self, workdir):
-        # the Build calls this just after __init__ and setDefaultWorkdir.
-        # ShellCommand and variants use a slave-side workdir, but some other
-        # steps do not. Subclasses which use a workdir should use the value
-        # set by this method unless they were constructed with something more
-        # specific.
+        # The Build calls this just after __init__().  ShellCommand
+        # and variants use a slave-side workdir, but some other steps
+        # do not. Subclasses which use a workdir should use the value
+        # set by this method unless they were constructed with
+        # something more specific.
         pass
 
     def addFactoryArguments(self, **kwargs):
         self.factory[1].update(kwargs)
 
     def getStepFactory(self):
         return self.factory
 
@@ -699,18 +704,19 @@ class BuildStep:
 
     def _startStep_2(self, res):
         if self.progress:
             self.progress.start()
         self.step_status.stepStarted()
         try:
             skip = self.start()
             if skip == SKIPPED:
-                reactor.callLater(0, self.releaseLocks)
-                reactor.callLater(0, self.deferred.callback, SKIPPED)
+                # this return value from self.start is a shortcut
+                # to finishing the step immediately
+                reactor.callLater(0, self.finished, SKIPPED)
         except:
             log.msg("BuildStep.startStep exception in .start")
             self.failed(Failure())
 
     def start(self):
         """Begin the step. Override this method and add code to do local
         processing, fire off remote commands, etc.
 
@@ -719,17 +725,16 @@ class BuildStep:
 
           c = RemoteCommandFoo(args)
           d = self.runCommand(c)
           d.addCallback(self.fooDone).addErrback(self.failed)
 
         As the step runs, it should send status information to the
         BuildStepStatus::
 
-          self.step_status.setColor('red')
           self.step_status.setText(['compile', 'failed'])
           self.step_status.setText2(['4', 'warnings'])
 
         To have some code parse stdio (or other log stream) in realtime, add
         a LogObserver subclass. This observer can use self.step.setProgress()
         to provide better progress notification to the step.::
 
           self.addLogObserver('stdio', MyLogObserver())
@@ -792,17 +797,16 @@ class BuildStep:
         log.msg("BuildStep.failed, traceback follows")
         log.err(why)
         try:
             if self.progress:
                 self.progress.finish()
             self.addHTMLLog("err.html", formatFailure(why))
             self.addCompleteLog("err.text", why.getTraceback())
             # could use why.getDetailedTraceback() for more information
-            self.step_status.setColor("purple")
             self.step_status.setText([self.name, "exception"])
             self.step_status.setText2([self.name])
             self.step_status.stepFinished(EXCEPTION)
         except:
             log.msg("exception during failure processing")
             log.err()
             # the progress stuff may still be whacked (the StepStatus may
             # think that it is still running), but the build overall will now
@@ -944,19 +948,19 @@ class LoggingBuildStep(BuildStep):
     def describe(self, done=False):
         raise NotImplementedError("implement this in a subclass")
 
     def startCommand(self, cmd, errorMessages=[]):
         """
         @param cmd: a suitable RemoteCommand which will be launched, with
                     all output being put into our self.stdio_log LogFile
         """
-        log.msg("ShellCommand.startCommand(cmd=%s)", (cmd,))
+        log.msg("ShellCommand.startCommand(cmd=%s)" % (cmd,))
+        log.msg("  cmd.args = %r" % (cmd.args))
         self.cmd = cmd # so we can interrupt it
-        self.step_status.setColor("yellow")
         self.step_status.setText(self.describe(False))
 
         # stdio is the first log
         self.stdio_log = stdio_log = self.addLog("stdio")
         cmd.useLog(stdio_log, True)
         for em in errorMessages:
             stdio_log.addHeader(em)
             # TODO: consider setting up self.stdio_log earlier, and have the
@@ -991,17 +995,16 @@ class LoggingBuildStep(BuildStep):
         # instead of FAILURE, might make the text a bit more clear.
         # 'reason' can be a Failure, or text
         self.addCompleteLog('interrupt', str(reason))
         d = self.cmd.interrupt(reason)
         return d
 
     def checkDisconnect(self, f):
         f.trap(error.ConnectionLost)
-        self.step_status.setColor("red")
         self.step_status.setText(self.describe(True) +
                                  ["failed", "slave", "lost"])
         self.step_status.setText2(["failed", "slave", "lost"])
         return self.finished(FAILURE)
 
     # to refine the status output, override one or more of the following
     # methods. Change as little as possible: start with the first ones on
     # this list and only proceed further if you have to    
@@ -1076,29 +1079,19 @@ class LoggingBuildStep(BuildStep):
                 return self.getText2(cmd, results)
         else:
             if (self.haltOnFailure or self.flunkOnFailure
                 or self.warnOnFailure):
                 # we're affecting the overall build, so tell them why
                 return self.getText2(cmd, results)
         return []
 
-    def getColor(self, cmd, results):
-        assert results in (SUCCESS, WARNINGS, FAILURE)
-        if results == SUCCESS:
-            return "green"
-        elif results == WARNINGS:
-            return "orange"
-        else:
-            return "red"
-
     def setStatus(self, cmd, results):
         # this is good enough for most steps, but it can be overridden to
         # get more control over the displayed text
-        self.step_status.setColor(self.getColor(cmd, results))
         self.step_status.setText(self.getText(cmd, results))
         self.step_status.setText2(self.maybeGetText2(cmd, results))
 
 # (WithProeprties used to be available in this module)
 from buildbot.process.properties import WithProperties
 _hush_pyflakes = [WithProperties]
 del _hush_pyflakes
 
--- a/buildbot/process/factory.py
+++ b/buildbot/process/factory.py
@@ -42,16 +42,18 @@ class BuildFactory(util.ComparableMixin)
 
     def addStep(self, step_or_factory, **kwargs):
         if isinstance(step_or_factory, BuildStep):
             s = step_or_factory.getStepFactory()
         else:
             s = (step_or_factory, dict(kwargs))
         self.steps.append(s)
 
+    def addSteps(self, steps):
+        self.steps.extend([ s.getStepFactory() for s in steps ])
 
 # BuildFactory subclasses for common build tools
 
 class GNUAutoconf(BuildFactory):
     def __init__(self, source, configure="./configure",
                  configureEnv={},
                  configureFlags=[],
                  compile=["make", "all"],
--- a/buildbot/process/properties.py
+++ b/buildbot/process/properties.py
@@ -13,17 +13,17 @@ class Properties(util.ComparableMixin):
 
     Objects of this class can be read like a dictionary -- in this case,
     only the property value is returned.
 
     As a special case, a property value of None is returned as an empty 
     string when used as a mapping.
     """
 
-    compare_attrs = ('properties')
+    compare_attrs = ('properties',)
 
     def __init__(self, **kwargs):
         """
         @param kwargs: initial property values (for testing)
         """
         self.properties = {}
         self.pmap = PropertyMap(self)
         if kwargs: self.update(kwargs, "TEST")
--- a/buildbot/process/step_twisted2.py
+++ b/buildbot/process/step_twisted2.py
@@ -136,26 +136,24 @@ class RunUnitTestsJelly(RunUnitTests):
         if count:
             result = (FAILURE, ["%d tes%s%s" % (count,
                                                 (count == 1 and 't' or 'ts'),
                                                 self.rtext(' (%s)'))])
         return self.stepComplete(result)
     def finishStatus(self, result):
         total = self.results.countTests()
         count = self.results.countFailures()
-        color = "green"
         text = []
         if count == 0:
             text.extend(["%d %s" % \
                          (total,
                           total == 1 and "test" or "tests"),
                          "passed"])
         else:
             text.append("tests")
             text.append("%d %s" % \
                         (count,
                          count == 1 and "failure" or "failures"))
-            color = "red"
-        self.updateCurrentActivity(color=color, text=text)
+        self.updateCurrentActivity(text=text)
         self.addFileToCurrentActivity("tests", self.results)
         #self.finishStatusSummary()
         self.finishCurrentActivity()
             
--- a/buildbot/scheduler.py
+++ b/buildbot/scheduler.py
@@ -83,20 +83,20 @@ class Scheduler(BaseUpstreamScheduler):
     called the C{treeStableTimer}, on a given set of Builders. It only pays
     attention to a single branch. You you can provide a C{fileIsImportant}
     function which will evaluate each Change to decide whether or not it
     should trigger a new build.
     """
 
     fileIsImportant = None
     compare_attrs = ('name', 'treeStableTimer', 'builderNames', 'branch',
-                     'fileIsImportant', 'properties')
+                     'fileIsImportant', 'properties', 'categories')
     
     def __init__(self, name, branch, treeStableTimer, builderNames,
-                 fileIsImportant=None, properties={}):
+                 fileIsImportant=None, properties={}, categories=None):
         """
         @param name: the name of this Scheduler
         @param branch: The branch name that the Scheduler should pay
                        attention to. Any Change that is not on this branch
                        will be ignored. It can be set to None to only pay
                        attention to the default branch.
         @param treeStableTimer: the duration, in seconds, for which the tree
                                 must remain unchanged before a build will be
@@ -111,16 +111,17 @@ class Scheduler(BaseUpstreamScheduler):
                                 worth building, and False if it is not.
                                 Unimportant Changes are accumulated until the
                                 build is triggered by an important change.
                                 The default value of None means that all
                                 Changes are important.
 
         @param properties: properties to apply to all builds started from this 
                            scheduler
+        @param categories: A list of categories of changes to accept
         """
 
         BaseUpstreamScheduler.__init__(self, name, properties)
         self.treeStableTimer = treeStableTimer
         errmsg = ("The builderNames= argument to Scheduler must be a list "
                   "of Builder description names (i.e. the 'name' key of the "
                   "Builder specification dictionary)")
         assert isinstance(builderNames, (list, tuple)), errmsg
@@ -131,29 +132,33 @@ class Scheduler(BaseUpstreamScheduler):
         if fileIsImportant:
             assert callable(fileIsImportant)
             self.fileIsImportant = fileIsImportant
 
         self.importantChanges = []
         self.unimportantChanges = []
         self.nextBuildTime = None
         self.timer = None
+        self.categories = categories
 
     def listBuilderNames(self):
         return self.builderNames
 
     def getPendingBuildTimes(self):
         if self.nextBuildTime is not None:
             return [self.nextBuildTime]
         return []
 
     def addChange(self, change):
         if change.branch != self.branch:
             log.msg("%s ignoring off-branch %s" % (self, change))
             return
+        if self.categories is not None and change.category not in self.categories:
+            log.msg("%s ignoring non-matching categories %s" % (self, change))
+            return
         if not self.fileIsImportant:
             self.addImportantChange(change)
         elif self.fileIsImportant(change):
             self.addImportantChange(change)
         else:
             self.addUnimportantChange(change)
 
     def addImportantChange(self, change):
@@ -167,17 +172,17 @@ class Scheduler(BaseUpstreamScheduler):
         log.msg("%s: change is not important, adding %s" % (self, change))
         self.unimportantChanges.append(change)
 
     def setTimer(self, when):
         log.msg("%s: setting timer to %s" %
                 (self, time.strftime("%H:%M:%S", time.localtime(when))))
         now = util.now()
         if when < now:
-            when = now + 1
+            when = now
         if self.timer:
             self.timer.cancel()
         self.timer = reactor.callLater(when - now, self.fireTimer)
 
     def stopTimer(self):
         if self.timer:
             self.timer.cancel()
             self.timer = None
@@ -270,16 +275,21 @@ class AnyBranchScheduler(BaseUpstreamSch
 
     def getPendingBuildTimes(self):
         bts = []
         for s in self.schedulers.values():
             if s.nextBuildTime is not None:
                 bts.append(s.nextBuildTime)
         return bts
 
+    def buildSetFinished(self, bss):
+        # we don't care if a build has finished; one of the per-branch builders
+        # will take care of it, instead.
+        pass
+
     def addChange(self, change):
         branch = change.branch
         if self.branches is not None and branch not in self.branches:
             log.msg("%s ignoring off-branch %s" % (self, change))
             return
         s = self.schedulers.get(branch)
         if not s:
             if branch:
@@ -287,27 +297,29 @@ class AnyBranchScheduler(BaseUpstreamSch
             else:
                 name = self.name + ".<default>"
             s = self.schedulerFactory(name, branch,
                                       self.treeStableTimer,
                                       self.builderNames,
                                       self.fileIsImportant)
             s.successWatchers = self.successWatchers
             s.setServiceParent(self)
+            s.properties = self.properties
             # TODO: does this result in schedulers that stack up forever?
             # When I make the persistify-pass, think about this some more.
             self.schedulers[branch] = s
         s.addChange(change)
 
 
 class Dependent(BaseUpstreamScheduler):
     """This scheduler runs some set of 'downstream' builds when the
     'upstream' scheduler has completed successfully."""
+    implements(interfaces.IDownstreamScheduler)
 
-    compare_attrs = ('name', 'upstream', 'builders', 'properties')
+    compare_attrs = ('name', 'upstream', 'builderNames', 'properties')
 
     def __init__(self, name, upstream, builderNames, properties={}):
         assert interfaces.IUpstreamScheduler.providedBy(upstream)
         BaseUpstreamScheduler.__init__(self, name, properties)
         self.upstream = upstream
         self.builderNames = builderNames
 
     def listBuilderNames(self):
@@ -326,17 +338,37 @@ class Dependent(BaseUpstreamScheduler):
         self.upstream.unsubscribeToSuccessfulBuilds(self.upstreamBuilt)
         return d
 
     def upstreamBuilt(self, ss):
         bs = buildset.BuildSet(self.builderNames, ss,
                     properties=self.properties)
         self.submitBuildSet(bs)
 
+    def checkUpstreamScheduler(self):
+        # find our *active* upstream scheduler (which may not be self.upstream!) by name
+        up_name = self.upstream.name
+        upstream = None
+        for s in self.parent.allSchedulers():
+            if s.name == up_name and interfaces.IUpstreamScheduler.providedBy(s):
+                upstream = s
+        if not upstream:
+            log.msg("ERROR: Couldn't find upstream scheduler of name <%s>" %
+                up_name)
 
+        # if it's already correct, we're good to go
+        if upstream is self.upstream:
+            return
+
+        # otherwise, associate with the new upstream.  We also keep listening
+        # to the old upstream, in case it's in the middle of a build
+        upstream.subscribeToSuccessfulBuilds(self.upstreamBuilt)
+        self.upstream = upstream
+        log.msg("Dependent <%s> connected to new Upstream <%s>" %
+                (self.name, up_name))
 
 class Periodic(BaseUpstreamScheduler):
     """Instead of watching for Changes, this Scheduler can just start a build
     at fixed intervals. The C{periodicBuildTimer} parameter sets the number
     of seconds to wait between such periodic builds. The first build will be
     run immediately."""
 
     # TODO: consider having this watch another (changed-based) scheduler and
@@ -404,41 +436,58 @@ class Nightly(BaseUpstreamScheduler):
 
      s = Nightly('SleighPreflightCheck', ['flying_circuits', 'radar'],
                  month=12, dayOfMonth=24, hour=12, minute=0)
 
     For dayOfWeek and dayOfMonth, builds are triggered if the date matches
     either of them. All time values are compared against the tuple returned
     by time.localtime(), so month and dayOfMonth numbers start at 1, not
     zero. dayOfWeek=0 is Monday, dayOfWeek=6 is Sunday.
+
+    onlyIfChanged functionality
+    s = Nightly('nightly', ['builder1', 'builder2'],
+                hour=3, minute=0, onlyIfChanged=True)
+    When the flag is True (False by default), the build is trigged if
+    the date matches and if the branch has changed
+
+    fileIsImportant parameter is implemented as defined in class Scheduler
     """
 
     compare_attrs = ('name', 'builderNames',
                      'minute', 'hour', 'dayOfMonth', 'month',
-                     'dayOfWeek', 'branch', 'properties')
+                     'dayOfWeek', 'branch', 'onlyIfChanged',
+                     'fileIsImportant', 'properties')
 
     def __init__(self, name, builderNames, minute=0, hour='*',
                  dayOfMonth='*', month='*', dayOfWeek='*',
-                 branch=None, properties={}):
+                 branch=None, fileIsImportant=None, onlyIfChanged=False, properties={}):
         # Setting minute=0 really makes this an 'Hourly' scheduler. This
         # seemed like a better default than minute='*', which would result in
         # a build every 60 seconds.
         BaseUpstreamScheduler.__init__(self, name, properties)
         self.builderNames = builderNames
         self.minute = minute
         self.hour = hour
         self.dayOfMonth = dayOfMonth
         self.month = month
         self.dayOfWeek = dayOfWeek
         self.branch = branch
+        self.onlyIfChanged = onlyIfChanged
         self.delayedRun = None
         self.nextRunTime = None
         self.reason = ("The Nightly scheduler named '%s' triggered this build"
                        % name)
 
+        self.importantChanges   = [] 
+        self.unimportantChanges = [] 
+        self.fileIsImportant    = None 
+        if fileIsImportant: 
+            assert callable(fileIsImportant) 
+            self.fileIsImportant = fileIsImportant 
+
     def addTime(self, timetuple, secs):
         return time.localtime(time.mktime(timetuple)+secs)
     def findFirstValueAtLeast(self, values, value, default=None):
         for v in values:
             if v >= value: return v
         return default
 
     def setTimer(self):
@@ -518,26 +567,61 @@ class Nightly(BaseUpstreamScheduler):
         # that
         if self.nextRunTime is None: return []
         return [self.nextRunTime]
 
     def doPeriodicBuild(self):
         # Schedule the next run
         self.setTimer()
 
-        # And trigger a build
-        bs = buildset.BuildSet(self.builderNames,
-                               SourceStamp(branch=self.branch),
-                               self.reason,
-                               properties=self.properties)
-        self.submitBuildSet(bs)
+        if  self.onlyIfChanged:
+            if len(self.importantChanges) > 0: 
+                changes = self.importantChanges + self.unimportantChanges 
+                # And trigger a build 
+                log.msg("Nightly Scheduler <%s>: triggering build" % self.name) 
+                bs = buildset.BuildSet(self.builderNames,
+                                       SourceStamp(changes=changes),
+                                       self.reason,
+                                       properties=self.properties)
+                self.submitBuildSet(bs)
+                # Reset the change lists 
+                self.importantChanges = [] 
+                self.unimportantChanges = []
+            else: 
+                log.msg("Nightly Scheduler <%s>: skipping build - No important change" % self.name)
+        else:
+            # And trigger a build
+            bs = buildset.BuildSet(self.builderNames,
+                                   SourceStamp(branch=self.branch),
+                                   self.reason,
+                                   properties=self.properties)
+            self.submitBuildSet(bs) 
 
     def addChange(self, change):
-        pass
-
+        if  self.onlyIfChanged:
+            if change.branch != self.branch: 
+                log.msg("Nightly Scheduler <%s>: ignoring change %d on off-branch %s" % (self.name, change.revision, change.branch)) 
+                return 
+            if not self.fileIsImportant:
+                self.addImportantChange(change) 
+            elif self.fileIsImportant(change): 
+                self.addImportantChange(change) 
+            else: 
+                self.addUnimportantChange(change)
+        else:
+            log.msg("Nightly Scheduler <%s>: no add change" % self.name)
+            pass
+ 
+    def addImportantChange(self, change):
+        log.msg("Nightly Scheduler <%s>: change %s from %s is important, adding it" % (self.name, change.revision, change.who)) 
+        self.importantChanges.append(change) 
+ 
+    def addUnimportantChange(self, change):
+        log.msg("Nightly Scheduler <%s>: change %s from %s is not important, adding it" % (self.name, change.revision, change.who)) 
+        self.unimportantChanges.append(change) 
 
 
 class TryBase(BaseScheduler):
     def __init__(self, name, builderNames, properties={}):
         BaseScheduler.__init__(self, name, properties)
         self.builderNames = builderNames
 
     def listBuilderNames(self):
@@ -546,16 +630,31 @@ class TryBase(BaseScheduler):
     def getPendingBuildTimes(self):
         # we can't predict what the developers are going to do in the future
         return []
 
     def addChange(self, change):
         # Try schedulers ignore Changes
         pass
 
+    def processBuilderList(self, builderNames):
+        # self.builderNames is the configured list of builders
+        # available for try.  If the user supplies a list of builders,
+        # it must be restricted to the configured list.  If not, build
+        # on all of the configured builders.
+        if builderNames:
+            for b in builderNames:
+                if not b in self.builderNames:
+                    log.msg("%s got with builder %s" % (self, b))
+                    log.msg(" but that wasn't in our list: %s"
+                            % (self.builderNames,))
+                    return []
+        else:
+            builderNames = self.builderNames
+        return builderNames
 
 class BadJobfile(Exception):
     pass
 
 class JobFileScanner(basic.NetstringReceiver):
     def __init__(self):
         self.strings = []
         self.transport = self # so transport.loseConnection works
@@ -628,28 +727,20 @@ class Try_Jobdir(TryBase):
             f = open(path, "r")
 
         try:
             builderNames, ss, bsid = self.parseJob(f)
         except BadJobfile:
             log.msg("%s reports a bad jobfile in %s" % (self, filename))
             log.err()
             return
-        # compare builderNames against self.builderNames
-        # TODO: think about this some more.. why bother restricting it?
-        # perhaps self.builderNames should be used as the default list
-        # instead of being used as a restriction?
-        for b in builderNames:
-            if not b in self.builderNames:
-                log.msg("%s got jobfile %s with builder %s" % (self,
-                                                               filename, b))
-                log.msg(" but that wasn't in our list: %s"
-                        % (self.builderNames,))
-                return
-
+        # Validate/fixup the builder names.
+        builderNames = self.processBuilderList(builderNames)
+        if not builderNames:
+            return
         reason = "'try' job"
         bs = buildset.BuildSet(builderNames, ss, reason=reason, 
                     bsid=bsid, properties=self.properties)
         self.submitBuildSet(bs)
 
 class Try_Userpass(TryBase):
     compare_attrs = ( 'name', 'builderNames', 'port', 'userpass', 'properties' )
     implements(portal.IRealm)
@@ -683,22 +774,20 @@ class Try_Userpass(TryBase):
 class Try_Userpass_Perspective(pbutil.NewCredPerspective):
     def __init__(self, parent, username):
         self.parent = parent
         self.username = username
 
     def perspective_try(self, branch, revision, patch, builderNames, properties={}):
         log.msg("user %s requesting build on builders %s" % (self.username,
                                                              builderNames))
-        for b in builderNames:
-            if not b in self.parent.builderNames:
-                log.msg("%s got job with builder %s" % (self, b))
-                log.msg(" but that wasn't in our list: %s"
-                        % (self.parent.builderNames,))
-                return
+        # Validate/fixup the builder names.
+        builderNames = self.parent.processBuilderList(builderNames)
+        if not builderNames:
+            return
         ss = SourceStamp(branch, revision, patch)
         reason = "'try' job from user %s" % self.username
 
         # roll the specified props in with our inherited props
         combined_props = Properties()
         combined_props.updateFromProperties(self.parent.properties)
         combined_props.update(properties, "try build")
 
--- a/buildbot/scripts/checkconfig.py
+++ b/buildbot/scripts/checkconfig.py
@@ -26,16 +26,17 @@ class ConfigLoader(master.BuildMaster):
         try:
             os.chdir(tempdir)
             # Add the temp directory to the library path so local modules work
             sys.path.append(tempdir)
             configFile = open(configFileName, "r")
             self.loadConfig(configFile)
         except:
             os.chdir(dir)
+            configFile.close()
             rmtree(tempdir)
             raise
         os.chdir(dir)
         rmtree(tempdir)
 
 if __name__ == '__main__':
     try:
         if len(sys.argv) > 1:
--- a/buildbot/scripts/logwatcher.py
+++ b/buildbot/scripts/logwatcher.py
@@ -39,17 +39,17 @@ class LogWatcher(LineOnlyReceiver):
 
     def start(self):
         # return a Deferred that fires when the reconfig process has
         # finished. It errbacks with TimeoutError if the finish line has not
         # been seen within 10 seconds, and with ReconfigError if the error
         # line was seen. If the logfile could not be opened, it errbacks with
         # an IOError.
         self.p = reactor.spawnProcess(self.pp, "/usr/bin/tail",
-                                      ("tail", "-F", "-n", "0", self.logfile),
+                                      ("tail", "-f", "-n", "0", self.logfile),
                                       env=os.environ,
                                       )
         self.running = True
         d = defer.maybeDeferred(self._start)
         return d
 
     def _start(self):
         self.d = defer.Deferred()
--- a/buildbot/scripts/runner.py
+++ b/buildbot/scripts/runner.py
@@ -314,39 +314,64 @@ def upgradeMaster(config):
 
 class MasterOptions(MakerBase):
     optFlags = [
         ["force", "f",
          "Re-use an existing directory (will not overwrite master.cfg file)"],
         ]
     optParameters = [
         ["config", "c", "master.cfg", "name of the buildmaster config file"],
+        ["log-size", "s", "1000000",
+         "size at which to rotate twisted log files"],
+        ["log-count", "l", "None",
+         "limit the number of kept old twisted log files"],
         ]
     def getSynopsis(self):
         return "Usage:    buildbot create-master [options] <basedir>"
 
     longdesc = """
     This command creates a buildmaster working directory and buildbot.tac
     file. The master will live in <dir> and create various files there.
 
     At runtime, the master will read a configuration file (named
     'master.cfg' by default) in its basedir. This file should contain python
     code which eventually defines a dictionary named 'BuildmasterConfig'.
     The elements of this dictionary are used to configure the Buildmaster.
     See doc/config.xhtml for details about what can be controlled through
     this interface."""
 
+    def postOptions(self):
+        MakerBase.postOptions(self)
+        if not re.match('^\d+$', self['log-size']):
+            raise usage.UsageError("log-size parameter needs to be an int")
+        if not re.match('^\d+$', self['log-count']) and \
+                self['log-count'] != 'None':
+            raise usage.UsageError("log-count parameter needs to be an int "+
+                                   " or None")
+
+
 masterTAC = """
 from twisted.application import service
 from buildbot.master import BuildMaster
 
 basedir = r'%(basedir)s'
 configfile = r'%(config)s'
+rotateLength = %(log-size)s
+maxRotatedFiles = %(log-count)s
 
 application = service.Application('buildmaster')
+try:
+  from twisted.python.logfile import LogFile
+  from twisted.python.log import ILogObserver, FileLogObserver
+  logfile = LogFile.fromFullPath("twistd.log", rotateLength=rotateLength,
+                                 maxRotatedFiles=maxRotatedFiles)
+  application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+except ImportError:
+  # probably not yet twisted 8.2.0 and beyond, can't set log yet
+  pass
 BuildMaster(basedir, configfile).setServiceParent(application)
 
 """
 
 def createMaster(config):
     m = Maker(config)
     m.mkdir()
     m.chdir()
@@ -369,20 +394,26 @@ class SlaveOptions(MakerBase):
 #        ["name", "n", None, "Name for this build slave"],
 #        ["passwd", "p", None, "Password for this build slave"],
 #        ["basedir", "d", ".", "Base directory to use"],
 #        ["master", "m", "localhost:8007",
 #         "Location of the buildmaster (host:port)"],
 
         ["keepalive", "k", 600,
          "Interval at which keepalives should be sent (in seconds)"],
-        ["usepty", None, 1,
-         "(1 or 0) child processes should be run in a pty"],
+        ["usepty", None, 0,
+         "(1 or 0) child processes should be run in a pty (default 0)"],
         ["umask", None, "None",
          "controls permissions of generated files. Use --umask=022 to be world-readable"],
+        ["maxdelay", None, 300,
+         "Maximum time between connection attempts"],
+        ["log-size", "s", "1000000",
+         "size at which to rotate twisted log files"],
+        ["log-count", "l", "None",
+         "limit the number of kept old twisted log files"],
         ]
     
     longdesc = """
     This command creates a buildslave working directory and buildbot.tac
     file. The bot will use the <name> and <passwd> arguments to authenticate
     itself when connecting to the master. All commands are run in a
     build-specific subdirectory of <basedir>. <master> is a string of the
     form 'hostname:port', and specifies where the buildmaster can be reached.
@@ -402,35 +433,54 @@ class SlaveOptions(MakerBase):
         self['master'] = master
         self['name'] = name
         self['passwd'] = passwd
 
     def postOptions(self):
         MakerBase.postOptions(self)
         self['usepty'] = int(self['usepty'])
         self['keepalive'] = int(self['keepalive'])
+        self['maxdelay'] = int(self['maxdelay'])
         if self['master'].find(":") == -1:
             raise usage.UsageError("--master must be in the form host:portnum")
+        if not re.match('^\d+$', self['log-size']):
+            raise usage.UsageError("log-size parameter needs to be an int")
+        if not re.match('^\d+$', self['log-count']) and \
+                self['log-count'] != 'None':
+            raise usage.UsageError("log-count parameter needs to be an int "+
+                                   " or None")
 
 slaveTAC = """
 from twisted.application import service
 from buildbot.slave.bot import BuildSlave
 
 basedir = r'%(basedir)s'
 buildmaster_host = '%(host)s'
 port = %(port)d
 slavename = '%(name)s'
 passwd = '%(passwd)s'
 keepalive = %(keepalive)d
 usepty = %(usepty)d
 umask = %(umask)s
+maxdelay = %(maxdelay)d
+rotateLength = %(log-size)s
+maxRotatedFiles = %(log-count)s
 
 application = service.Application('buildslave')
+try:
+  from twisted.python.logfile import LogFile
+  from twisted.python.log import ILogObserver, FileLogObserver
+  logfile = LogFile.fromFullPath("twistd.log", rotateLength=rotateLength,
+                                 maxRotatedFiles=maxRotatedFiles)
+  application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+except ImportError:
+  # probably not yet twisted 8.2.0 and beyond, can't set log yet
+  pass
 s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
-               keepalive, usepty, umask=umask)
+               keepalive, usepty, umask=umask, maxdelay=maxdelay)
 s.setServiceParent(application)
 
 """
 
 def createSlave(config):
     m = Maker(config)
     m.mkdir()
     m.chdir()
@@ -662,16 +712,17 @@ def statusgui(config):
     c.run()
 
 class SendChangeOptions(usage.Options):
     optParameters = [
         ("master", "m", None,
          "Location of the buildmaster's PBListener (host:port)"),
         ("username", "u", None, "Username performing the commit"),
         ("branch", "b", None, "Branch specifier"),
+        ("category", "c", None, "Category of repository"),
         ("revision", "r", None, "Revision specifier (string)"),
         ("revision_number", "n", None, "Revision specifier (integer)"),
         ("revision_file", None, None, "Filename containing revision spec"),
         ("comments", "m", None, "log message"),
         ("logfile", "F", None,
          "Read the log messages from this file (- for stdin)"),
         ]
     def getSynopsis(self):
@@ -684,16 +735,17 @@ def sendchange(config, runReactor=False)
     """Send a single change to the buildmaster's PBChangeSource. The
     connection will be drpoped as soon as the Change has been sent."""
     from buildbot.clients.sendchange import Sender
 
     opts = loadOptions()
     user = config.get('username', opts.get('username'))
     master = config.get('master', opts.get('master'))
     branch = config.get('branch', opts.get('branch'))
+    category = config.get('category', opts.get('category'))
     revision = config.get('revision')
     # SVN and P4 use numeric revisions
     if config.get("revision_number"):
         revision = int(config['revision_number'])
     if config.get("revision_file"):
         revision = open(config["revision_file"],"r").read()
 
     comments = config.get('comments')
@@ -707,17 +759,17 @@ def sendchange(config, runReactor=False)
         comments = ""
 
     files = config.get('files', [])
 
     assert user, "you must provide a username"
     assert master, "you must provide the master location"
 
     s = Sender(master, user)
-    d = s.send(branch, revision, comments, files)
+    d = s.send(branch, revision, comments, files, category=category)
     if runReactor:
         d.addCallbacks(s.printSuccess, s.printFailure)
         d.addBoth(s.stop)
         s.run()
     return d
 
 
 class ForceOptions(usage.Options):
@@ -772,16 +824,17 @@ class TryOptions(usage.Options):
         ["builder", "b", None,
          "Run the trial build on this Builder. Can be used multiple times."],
         ["properties", None, None,
          "A set of properties made available in the build environment, format:prop=value,propb=valueb..."],
         ]
 
     optFlags = [
         ["wait", None, "wait until the builds have finished"],
+        ["dryrun", 'n', "Gather info, but don't actually submit."],
         ]
 
     def __init__(self):
         super(TryOptions, self).__init__()
         self['builders'] = []
         self['properties'] = {}
 
     def opt_builder(self, option):
--- a/buildbot/scripts/sample.cfg
+++ b/buildbot/scripts/sample.cfg
@@ -12,18 +12,18 @@
 
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}
 
 ####### BUILDSLAVES
 
 # the 'slaves' list defines the set of allowable buildslaves. Each element is
-# a tuple of bot-name and bot-password. These correspond to values given to
-# the buildslave's mktap invocation.
+# a BuildSlave object, which is created with bot-name, bot-password.  These
+# correspond to values given to the buildslave's mktap invocation.
 from buildbot.buildslave import BuildSlave
 c['slaves'] = [BuildSlave("bot1name", "bot1passwd")]
 
 # to limit to two concurrent builds on a slave, use
 #  c['slaves'] = [BuildSlave("bot1name", "bot1passwd", max_builds=2)]
 
 
 # 'slavePortnum' defines the TCP port to listen on. This must match the value
@@ -71,18 +71,18 @@ c['schedulers'].append(Scheduler(name="a
                                  treeStableTimer=2*60,
                                  builderNames=["buildbot-full"]))
 
 
 ####### BUILDERS
 
 # the 'builders' list defines the Builders. Each one is configured with a
 # dictionary, using the following keys:
-#  name (required): the name used to describe this bilder
-#  slavename (required): which slave to use, must appear in c['bots']
+#  name (required): the name used to describe this builder
+#  slavename (required): which slave to use (must appear in c['bots'])
 #  builddir (required): which subdirectory to run the builder in
 #  factory (required): a BuildFactory to define how the build is run
 #  periodicBuildTime (optional): if set, force a build every N seconds
 
 # buildbot/process/factory.py provides several BuildFactory classes you can
 # start with, which implement build processes for common targets (GNU
 # autoconf projects, CPAN perl modules, etc). The factory.BuildFactory is the
 # base class, and is configured with a series of BuildSteps. When the build
@@ -136,17 +136,17 @@ c['status'].append(html.WebStatus(http_p
 # c['status'].append(client.PBListener(9988))
 
 
 ####### DEBUGGING OPTIONS
 
 # if you set 'debugPassword', then you can connect to the buildmaster with
 # the diagnostic tool in contrib/debugclient.py . From this tool, you can
 # manually force builds and inject changes, which may be useful for testing
-# your buildmaster without actually commiting changes to your repository (or
+# your buildmaster without actually committing changes to your repository (or
 # before you have a functioning 'sources' set up). The debug tool uses the
 # same port number as the slaves do: 'slavePortnum'.
 
 #c['debugPassword'] = "debugpassword"
 
 # if you set 'manhole', you can ssh into the buildmaster and get an
 # interactive python shell, which may be useful for debugging buildbot
 # internals. It is probably only useful for buildbot developers. You can also
--- a/buildbot/scripts/tryclient.py
+++ b/buildbot/scripts/tryclient.py
@@ -207,26 +207,62 @@ class GitExtractor(SourceStampExtractor)
     patchlevel = 1
     vcexe = "git"
 
     def getBaseRevision(self):
         d = self.dovc(["branch", "--no-color", "-v", "--no-abbrev"])
         d.addCallback(self.parseStatus)
         return d
 
+    def readConfig(self):
+        d = self.dovc(["config", "-l"])
+        d.addCallback(self.parseConfig)
+        return d
+
+    def parseConfig(self, res):
+        git_config = {}
+        for l in res.split("\n"):
+            if l.strip():
+                parts = l.strip().split("=", 2)
+                git_config[parts[0]] = parts[1]
+
+        # If we're tracking a remote, consider that the base.
+        remote = git_config.get("branch." + self.branch + ".remote")
+        ref = git_config.get("branch." + self.branch + ".merge")
+        if remote and ref:
+            remote_branch = ref.split("/", 3)[-1]
+            d = self.dovc(["rev-parse", remote + "/" + remote_branch])
+            d.addCallback(self.override_baserev)
+            return d
+
+    def override_baserev(self, res):
+        self.baserev = res.strip()
+
     def parseStatus(self, res):
         # The current branch is marked by '*' at the start of the
         # line, followed by the branch name and the SHA1.
         #
         # Branch names may contain pretty much anything but whitespace.
         m = re.search(r'^\* (\S+)\s+([0-9a-f]{40})', res, re.MULTILINE)
         if m:
-            self.branch = m.group(1)
             self.baserev = m.group(2)
-            return
+            # If a branch is specified, parse out the rev it points to
+            # and extract the local name (assuming it has a slash).
+            # This may break if someone specifies the name of a local
+            # branch that has a slash in it and has no corresponding
+            # remote branch (or something similarly contrived).
+            if self.branch:
+                d = self.dovc(["rev-parse", self.branch])
+                if '/' in self.branch:
+                    self.branch = self.branch.split('/', 1)[1]
+                d.addCallback(self.override_baserev)
+                return d
+            else:
+                self.branch = m.group(1)
+                return self.readConfig()
         raise IndexError("Could not find current GIT branch: %s" % res)
 
     def getPatch(self, res):
         d = self.dovc(["diff", self.baserev])
         d.addCallback(self.readPatch, self.patchlevel)
         return d
 
 def getSourceStamp(vctype, treetop, branch=None):
@@ -343,18 +379,16 @@ class Try(pb.Referenceable):
     quiet = False
 
     def __init__(self, config):
         self.config = config
         self.opts = runner.loadOptions()
         self.connect = self.getopt('connect', 'try_connect')
         assert self.connect, "you must specify a connect style: ssh or pb"
         self.builderNames = self.getopt('builders', 'try_builders')
-        assert self.builderNames, "no builders! use --builder or " \
-               "try_builders=[names..] in .buildbot/options"
 
     def getopt(self, config_name, options_name, default=None):
         value = self.config.get(config_name)
         if value is None or value == []:
             value = self.opts.get(options_name)
         if value is None or value == []:
             value = default
         return value
@@ -404,16 +438,28 @@ class Try(pb.Referenceable):
             revspec = ss.revision
             if revspec is None:
                 revspec = ""
             self.jobfile = createJobfile(self.bsid,
                                          ss.branch or "", revspec,
                                          patchlevel, diff,
                                          self.builderNames)
 
+    def fakeDeliverJob(self):
+        # Display the job to be delivered, but don't perform delivery.
+        ss = self.sourcestamp
+        print ("Job:\n\tBranch: %s\n\tRevision: %s\n\tBuilders: %s\n%s"
+               % (ss.branch,
+                  ss.revision,
+                  self.builderNames,
+                  ss.patch[1]))
+        d = defer.Deferred()
+        d.callback(True)
+        return d
+
     def deliverJob(self):
         # returns a Deferred that fires when the job has been delivered
         opts = self.opts
 
         if self.connect == "ssh":
             tryhost = self.getopt("tryhost", "try_host")
             tryuser = self.getopt("username", "try_username")
             trydir = self.getopt("trydir", "try_dir")
@@ -629,17 +675,20 @@ class Try(pb.Referenceable):
     def run(self):
         # we can't do spawnProcess until we're inside reactor.run(), so get
         # funky
         print "using '%s' connect method" % self.connect
         self.exitcode = 0
         d = defer.Deferred()
         d.addCallback(lambda res: self.createJob())
         d.addCallback(lambda res: self.announce("job created"))
-        d.addCallback(lambda res: self.deliverJob())
+        deliver = self.deliverJob
+        if bool(self.config.get("dryrun")):
+            deliver = self.fakeDeliverJob
+        d.addCallback(lambda res: deliver())
         d.addCallback(lambda res: self.announce("job has been delivered"))
         d.addCallback(lambda res: self.getStatus())
         d.addErrback(log.err)
         d.addCallback(self.cleanup)
         d.addCallback(lambda res: reactor.stop())
 
         reactor.callLater(0, d.callback, None)
         reactor.run()
--- a/buildbot/slave/bot.py
+++ b/buildbot/slave/bot.py
@@ -352,24 +352,29 @@ class BotFactory(ReconnectingPBClientFac
 
     # 'keepaliveTimeout' seconds before the interval expires, we will send a
     # keepalive request, both to add some traffic to the connection, and to
     # prompt a response from the master in case all our builders are idle. We
     # don't insist upon receiving a timely response from this message: a slow
     # link might put the request at the wrong end of a large build message.
     keepaliveTimeout = 30 # how long we will go without a response
 
+    # 'maxDelay' determines the maximum amount of time the slave will wait
+    # between connection retries
+    maxDelay = 300
+
     keepaliveTimer = None
     activityTimer = None
     lastActivity = 0
     unsafeTracebacks = 1
     perspective = None
 
-    def __init__(self, keepaliveInterval, keepaliveTimeout):
+    def __init__(self, keepaliveInterval, keepaliveTimeout, maxDelay):
         ReconnectingPBClientFactory.__init__(self)
+        self.maxDelay = maxDelay
         self.keepaliveInterval = keepaliveInterval
         self.keepaliveTimeout = keepaliveTimeout
 
     def startedConnecting(self, connector):
         ReconnectingPBClientFactory.startedConnecting(self, connector)
         self.connector = connector
 
     def gotPerspective(self, perspective):
@@ -461,27 +466,27 @@ class BuildSlave(service.MultiService):
     # returning. The DelayedCalls used to implement this are stashed in the
     # list so they can be cancelled later.
 
     # debugOpts['failPingOnce'] can be set to True to make the slaveping fail
     # exactly once.
 
     def __init__(self, buildmaster_host, port, name, passwd, basedir,
                  keepalive, usePTY, keepaliveTimeout=30, umask=None,
-                 debugOpts={}):
+                 maxdelay=300, debugOpts={}):
         log.msg("Creating BuildSlave -- buildbot.version: %s" % buildbot.version)
         service.MultiService.__init__(self)
         self.debugOpts = debugOpts.copy()
         bot = self.botClass(basedir, usePTY)
         bot.setServiceParent(self)
         self.bot = bot
         if keepalive == 0:
             keepalive = None
         self.umask = umask
-        bf = self.bf = BotFactory(keepalive, keepaliveTimeout)
+        bf = self.bf = BotFactory(keepalive, keepaliveTimeout, maxdelay)
         bf.startLogin(credentials.UsernamePassword(name, passwd), client=bot)
         self.connection = c = internet.TCPClient(buildmaster_host, port, bf)
         c.setServiceParent(self)
 
     def waitUntilDisconnected(self):
         # utility method for testing. Returns a Deferred that will fire when
         # we lose the connection to the master.
         if not self.bf.perspective:
--- a/buildbot/slave/commands.py
+++ b/buildbot/slave/commands.py
@@ -10,17 +10,17 @@ from twisted.python import log, failure,
 from twisted.python.procutils import which
 
 from buildbot.slave.interfaces import ISlaveCommand
 from buildbot.slave.registry import registerSlaveCommand
 
 # this used to be a CVS $-style "Revision" auto-updated keyword, but since I
 # moved to Darcs as the primary repository, this is updated manually each
 # time this file is changed. The last cvs_ver that was here was 1.51 .
-command_version = "2.5"
+command_version = "2.8"
 
 # version history:
 #  >=1.17: commands are interruptable
 #  >=1.28: Arch understands 'revision', added Bazaar
 #  >=1.33: Source classes understand 'retry'
 #  >=1.39: Source classes correctly handle changes in branch (except Git)
 #          Darcs accepts 'revision' (now all do but Git) (well, and P4Sync)
 #          Arch/Baz should accept 'build-config'
@@ -32,22 +32,61 @@ command_version = "2.5"
 #          (not externally visible: ShellCommandPP has writeStdin/closeStdin.
 #          ShellCommand accepts new arguments (logfiles=, initialStdin=,
 #          keepStdinOpen=) and no longer accepts stdin=)
 #          (release 0.7.4)
 #  >= 2.2: added monotone, uploadFile, and downloadFile (release 0.7.5)
 #  >= 2.3: added bzr (release 0.7.6)
 #  >= 2.4: Git understands 'revision' and branches
 #  >= 2.5: workaround added for remote 'hg clone --rev REV' when hg<0.9.2
+#  >= 2.6: added uploadDirectory
+#  >= 2.7: added usePTY option to SlaveShellCommand
+#  >= 2.8: added username and password args to SVN class
 
 class CommandInterrupted(Exception):
     pass
 class TimeoutError(Exception):
     pass
 
+class Obfuscated:
+    """An obfuscated string in a command"""
+    def __init__(self, real, fake):
+        self.real = real
+        self.fake = fake
+
+    def __str__(self):
+        return self.fake
+
+    def __repr__(self):
+        return `self.fake`
+
+    def get_real(command):
+        rv = command
+        if type(command) == types.ListType:
+            rv = []
+            for elt in command:
+                if isinstance(elt, Obfuscated):
+                    rv.append(elt.real)
+                else:
+                    rv.append(elt)
+        return rv
+    get_real = staticmethod(get_real)
+
+    def get_fake(command):
+        rv = command
+        if type(command) == types.ListType:
+            rv = []
+            for elt in command:
+                if isinstance(elt, Obfuscated):
+                    rv.append(elt.fake)
+                else:
+                    rv.append(elt)
+        return rv
+    get_fake = staticmethod(get_fake)
+
 class AbandonChain(Exception):
     """A series of chained steps can raise this exception to indicate that
     one of the intermediate ShellCommands has failed, such that there is no
     point in running the remainder. 'rc' should be the non-zero exit code of
     the failing ShellCommand."""
 
     def __repr__(self):
         return "<AbandonChain rc=%s>" % self.args[0]
@@ -231,30 +270,33 @@ class ShellCommand:
     # http://www.opengroup.org/onlinepubs/000095399/functions/clock_getres.html
     # Then changes to the system clock during a run wouldn't effect the "elapsed
     # time" results.
 
     def __init__(self, builder, command,
                  workdir, environ=None,
                  sendStdout=True, sendStderr=True, sendRC=True,
                  timeout=None, initialStdin=None, keepStdinOpen=False,
-                 keepStdout=False, keepStderr=False,
-                 logfiles={}):
+                 keepStdout=False, keepStderr=False, logEnviron=True,
+                 logfiles={}, usePTY="slave-config"):
         """
 
         @param keepStdout: if True, we keep a copy of all the stdout text
                            that we've seen. This copy is available in
                            self.stdout, which can be read after the command
                            has finished.
         @param keepStderr: same, for stderr
 
+        @param usePTY: "slave-config" -> use the SlaveBuilder's usePTY;
+            otherwise, true to use a PTY, false to not use a PTY.
         """
 
         self.builder = builder
-        self.command = command
+        self.command = Obfuscated.get_real(command)
+        self.fake_command = Obfuscated.get_fake(command)
         self.sendStdout = sendStdout
         self.sendStderr = sendStderr
         self.sendRC = sendRC
         self.logfiles = logfiles
         self.workdir = workdir
         self.environ = os.environ.copy()
         if environ:
             if environ.has_key('PYTHONPATH'):
@@ -273,40 +315,45 @@ class ShellCommand:
                     # strings, so don't do that.
                     ppath = ppath + os.pathsep + self.environ['PYTHONPATH']
 
                 environ['PYTHONPATH'] = ppath
 
             self.environ.update(environ)
         self.initialStdin = initialStdin
         self.keepStdinOpen = keepStdinOpen
+        self.logEnviron = logEnviron
         self.timeout = timeout
         self.timer = None
         self.keepStdout = keepStdout
         self.keepStderr = keepStderr
 
+
+        if usePTY == "slave-config":
+            self.usePTY = self.builder.usePTY
+        else:
+            self.usePTY = usePTY
+
         # usePTY=True is a convenience for cleaning up all children and
-        # grandchildren of a hung command. Fall back to usePTY=False on
-        # systems where ptys cause problems.
-
-        self.usePTY = self.builder.usePTY
-        if runtime.platformType != "posix":
-            self.usePTY = False # PTYs are posix-only
-        if initialStdin is not None:
-            # for .closeStdin to matter, we must use a pipe, not a PTY
+        # grandchildren of a hung command. Fall back to usePTY=False on systems
+        # and in situations where ptys cause problems.  PTYs are posix-only,
+        # and for .closeStdin to matter, we must use a pipe, not a PTY
+        if runtime.platformType != "posix" or initialStdin is not None:
+            if self.usePTY and usePTY != "slave-config":
+                self.sendStatus({'header': "WARNING: disabling usePTY for this command"})
             self.usePTY = False
 
         self.logFileWatchers = []
         for name,filename in self.logfiles.items():
             w = LogFileWatcher(self, name,
                                os.path.join(self.workdir, filename))
             self.logFileWatchers.append(w)
 
     def __repr__(self):
-        return "<slavecommand.ShellCommand '%s'>" % self.command
+        return "<slavecommand.ShellCommand '%s'>" % self.fake_command
 
     def sendStatus(self, status):
         self.builder.sendUpdate(status)
 
     def start(self):
         # return a Deferred which fires (with the exit code) when the command
         # completes
         if self.keepStdout:
@@ -325,70 +372,80 @@ class ShellCommand:
 
     def _startCommand(self):
         # ensure workdir exists
         if not os.path.isdir(self.workdir):
             os.makedirs(self.workdir)
         log.msg("ShellCommand._startCommand")
         if self.notreally:
             self.sendStatus({'header': "command '%s' in dir %s" % \
-                             (self.command, self.workdir)})
+                             (self.fake_command, self.workdir)})
             self.sendStatus({'header': "(not really)\n"})
             self.finished(None, 0)
             return
 
         self.pp = ShellCommandPP(self)
 
         if type(self.command) in types.StringTypes:
             if runtime.platformType  == 'win32':
-                argv = [os.environ['COMSPEC'], '/c', self.command]
+                argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args
+                if '/c' not in argv: argv += ['/c'] 
+                argv += [self.command]
             else:
                 # for posix, use /bin/sh. for other non-posix, well, doesn't
                 # hurt to try
                 argv = ['/bin/sh', '-c', self.command]
+            display = self.fake_command
         else:
             if runtime.platformType  == 'win32':
-                argv = [os.environ['COMSPEC'], '/c'] + list(self.command)
+                argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args
+                if '/c' not in argv: argv += ['/c'] 
+                argv += list(self.command)
             else:
                 argv = self.command
+            display = " ".join(self.fake_command)
+
+        # $PWD usually indicates the current directory; spawnProcess may not
+        # update this value, though, so we set it explicitly here.
+        self.environ['PWD'] = os.path.abspath(self.workdir)
 
         # self.stdin is handled in ShellCommandPP.connectionMade
 
         # first header line is the command in plain text, argv joined with
         # spaces. You should be able to cut-and-paste this into a shell to
         # obtain the same results. If there are spaces in the arguments, too
         # bad.
-        msg = " ".join(argv)
-        log.msg(" " + msg)
-        self.sendStatus({'header': msg+"\n"})
+        log.msg(" " + display)
+        self.sendStatus({'header': display+"\n"})
 
         # then comes the secondary information
         msg = " in dir %s" % (self.workdir,)
         if self.timeout:
             msg += " (timeout %d secs)" % (self.timeout,)
         log.msg(" " + msg)
         self.sendStatus({'header': msg+"\n"})
 
         msg = " watching logfiles %s" % (self.logfiles,)
         log.msg(" " + msg)
         self.sendStatus({'header': msg+"\n"})
 
-        # then the argv array for resolving unambiguity
-        msg = " argv: %s" % (argv,)
+        # then the obfuscated command array for resolving unambiguity
+        msg = " argv: %s" % (self.fake_command,)
         log.msg(" " + msg)
         self.sendStatus({'header': msg+"\n"})
 
         # then the environment, since it sometimes causes problems
-        msg = " environment:\n"
-        env_names = self.environ.keys()
-        env_names.sort()
-        for name in env_names:
-            msg += "  %s=%s\n" % (name, self.environ[name])
-        log.msg(" environment: %s" % (self.environ,))
-        self.sendStatus({'header': msg})
+        if self.logEnviron:
+            msg = " environment:\n"
+            env_names = self.environ.keys()
+            env_names.sort()
+            for name in env_names:
+                msg += "  %s=%s\n" % (name, self.environ[name])
+            log.msg(" environment: %s" % (self.environ,))
+            self.sendStatus({'header': msg})
 
         if self.initialStdin:
             msg = " writing %d bytes to stdin" % len(self.initialStdin)
             log.msg(" " + msg)
             self.sendStatus({'header': msg+"\n"})
 
         if self.keepStdinOpen:
             msg = " leaving stdin open"
@@ -851,16 +908,109 @@ class SlaveFileUploadCommand(Command):
             self.sendStatus({'rc': self.rc})
         else:
             self.sendStatus({'stderr': self.stderr, 'rc': self.rc})
         return res
 
 registerSlaveCommand("uploadFile", SlaveFileUploadCommand, command_version)
 
 
+class SlaveDirectoryUploadCommand(Command):
+    """
+    Upload a directory from slave to build master
+    Arguments:
+
+        - ['workdir']:   base directory to use
+        - ['slavesrc']:  name of the slave-side directory to read from
+        - ['writer']:    RemoteReference to a transfer._DirectoryWriter object
+        - ['maxsize']:   max size (in bytes) of file to write
+        - ['blocksize']: max size for each data block
+    """
+    debug = True
+
+    def setup(self, args):
+        self.workdir = args['workdir']
+        self.dirname = args['slavesrc']
+        self.writer = args['writer']
+        self.remaining = args['maxsize']
+        self.blocksize = args['blocksize']
+        self.stderr = None
+        self.rc = 0
+
+    def start(self):
+        if self.debug:
+            log.msg('SlaveDirectoryUploadCommand started')
+
+	# create some lists with all files and directories
+	foundFiles = []
+	foundDirs = []
+
+	self.baseRoot = os.path.join(self.builder.basedir,
+                                     self.workdir,
+                        	     os.path.expanduser(self.dirname))
+	if self.debug:
+	    log.msg("baseRoot: %r" % self.baseRoot)
+
+	for root, dirs, files in os.walk(self.baseRoot):
+	    tempRoot = root
+	    relRoot = ''
+	    while (tempRoot != self.baseRoot):
+	        tempRoot, tempRelRoot = os.path.split(tempRoot)
+	        relRoot = os.path.join(tempRelRoot, relRoot)
+	    for name in files:
+	        foundFiles.append(os.path.join(relRoot, name))
+	    for directory in dirs:
+	        foundDirs.append(os.path.join(relRoot, directory))
+
+	if self.debug:
+	    log.msg("foundDirs: %s" % (str(foundDirs)))
+	    log.msg("foundFiles: %s" % (str(foundFiles)))
+	
+	# create all directories on the master, to catch also empty ones
+	for dirname in foundDirs:
+	    self.writer.callRemote("createdir", dirname)
+
+	for filename in foundFiles:
+	    self._writeFile(filename)
+
+	return None
+
+    def _writeFile(self, filename):
+        """Write a file to the remote writer"""
+
+        log.msg("_writeFile: %r" % (filename))
+	self.writer.callRemote('open', filename)
+	data = open(os.path.join(self.baseRoot, filename), "r").read()
+	self.writer.callRemote('write', data)
+	self.writer.callRemote('close')
+        return None
+
+    def interrupt(self):
+        if self.debug:
+            log.msg('interrupted')
+        if self.interrupted:
+            return
+        if self.stderr is None:
+            self.stderr = 'Upload of %r interrupted' % self.path
+            self.rc = 1
+        self.interrupted = True
+        # the next _writeBlock call will notice the .interrupted flag
+
+    def finished(self, res):
+        if self.debug:
+            log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc))
+        if self.stderr is None:
+            self.sendStatus({'rc': self.rc})
+        else:
+            self.sendStatus({'stderr': self.stderr, 'rc': self.rc})
+        return res
+
+registerSlaveCommand("uploadDirectory", SlaveDirectoryUploadCommand, command_version)
+
+
 class SlaveFileDownloadCommand(Command):
     """
     Download a file from master to slave
     Arguments:
 
         - ['workdir']:   base directory to use
         - ['slavedest']: name of the slave-side file to be created
         - ['reader']:    RemoteReference to a transfer._FileReader object
@@ -1019,16 +1169,18 @@ class SlaveShellCommand(Command):
         - ['initial_stdin']: a string which will be written to the command's
                              stdin as soon as it starts
         - ['keep_stdin_open']: unless True, the command's stdin will be
                                closed as soon as initial_stdin has been
                                written. Set this to True if you plan to write
                                to stdin after the command has been started.
         - ['want_stdout']: 0 if stdout should be thrown away
         - ['want_stderr']: 0 if stderr should be thrown away
+        - ['usePTY']: True or False if the command should use a PTY (defaults to
+                      configuration of the slave)
         - ['not_really']: 1 to skip execution and return rc=0
         - ['timeout']: seconds of silence to tolerate before killing command
         - ['logfiles']: dict mapping LogFile name to the workdir-relative
                         filename of a local log file. This local file will be
                         watched just like 'tail -f', and all changes will be
                         written to 'log' status updates.
 
     ShellCommand creates the following status messages:
@@ -1049,16 +1201,17 @@ class SlaveShellCommand(Command):
                          workdir, environ=args.get('env'),
                          timeout=args.get('timeout', None),
                          sendStdout=args.get('want_stdout', True),
                          sendStderr=args.get('want_stderr', True),
                          sendRC=True,
                          initialStdin=args.get('initial_stdin'),
                          keepStdinOpen=args.get('keep_stdin_open'),
                          logfiles=args.get('logfiles', {}),
+                         usePTY=args.get('usePTY', "slave-config"),
                          )
         self.command = c
         d = self.command.start()
         return d
 
     def interrupt(self):
         self.interrupted = True
         self.command.kill("command interrupted")
@@ -1197,17 +1350,17 @@ class SourceBase(Command):
 
     sourcedata = ""
 
     def setup(self, args):
         # if we need to parse the output, use this environment. Otherwise
         # command output will be in whatever the buildslave's native language
         # has been set to.
         self.env = os.environ.copy()
-        self.env['LC_ALL'] = "C"
+        self.env['LC_MESSAGES'] = "C"
 
         self.workdir = args['workdir']
         self.mode = args.get('mode', "update")
         self.revision = args.get('revision')
         self.patch = args.get('patch')
         self.timeout = args.get('timeout', 120)
         self.retry = args.get('retry')
         # VC-specific subclasses should override this to extract more args.
@@ -1222,34 +1375,37 @@ class SourceBase(Command):
             self.srcdir = "source" # hardwired directory name, sorry
         else:
             self.srcdir = self.workdir
         self.sourcedatafile = os.path.join(self.builder.basedir,
                                            self.srcdir,
                                            ".buildbot-sourcedata")
 
         d = defer.succeed(None)
-        # do we need to clobber anything?
-        if self.mode in ("copy", "clobber", "export"):
-            d.addCallback(self.doClobber, self.workdir)
+        self.maybeClobber(d)
         if not (self.sourcedirIsUpdateable() and self.sourcedataMatches()):
             # the directory cannot be updated, so we have to clobber it.
             # Perhaps the master just changed modes from 'export' to
             # 'update'.
             d.addCallback(self.doClobber, self.srcdir)
 
         d.addCallback(self.doVC)
 
         if self.mode == "copy":
             d.addCallback(self.doCopy)
         if self.patch:
             d.addCallback(self.doPatch)
         d.addCallbacks(self._sendRC, self._checkAbandoned)
         return d
 
+    def maybeClobber(self, d):
+        # do we need to clobber anything?
+        if self.mode in ("copy", "clobber", "export"):
+            d.addCallback(self.doClobber, self.workdir)
+
     def interrupt(self):
         self.interrupted = True
         if self.command:
             self.command.kill("command interrupted")
 
     def doVC(self, res):
         if self.interrupted:
             raise AbandonChain(1)
@@ -1353,16 +1509,17 @@ class SourceBase(Command):
             delay, repeats = self.retry
             if repeats >= 0:
                 self.retry = (delay, repeats-1)
                 msg = ("update failed, trying %d more times after %d seconds"
                        % (repeats, delay))
                 self.sendStatus({'header': msg + "\n"})
                 log.msg(msg)
                 d = defer.Deferred()
+                self.maybeClobber(d)
                 d.addCallback(lambda res: self.doVCFull())
                 d.addBoth(self.maybeDoVCRetry)
                 reactor.callLater(delay, d.callback, None)
                 return d
         return res
 
     def doClobber(self, dummy, dirname):
         # TODO: remove the old tree in the background
@@ -1386,50 +1543,62 @@ class SourceBase(Command):
         d = os.path.join(self.builder.basedir, dirname)
         if runtime.platformType != "posix":
             # if we're running on w32, use rmtree instead. It will block,
             # but hopefully it won't take too long.
             rmdirRecursive(d)
             return defer.succeed(0)
         command = ["rm", "-rf", d]
         c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=0, timeout=self.timeout)
+                         sendRC=0, timeout=self.timeout, usePTY=False)
+
         self.command = c
         # sendRC=0 means the rm command will send stdout/stderr to the
         # master, but not the rc=0 when it finishes. That job is left to
         # _sendRC
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         return d
 
     def doCopy(self, res):
         # now copy tree to workdir
         fromdir = os.path.join(self.builder.basedir, self.srcdir)
         todir = os.path.join(self.builder.basedir, self.workdir)
         if runtime.platformType != "posix":
+            self.sendStatus({'header': "Since we're on a non-POSIX platform, "
+            "we're not going to try to execute cp in a subprocess, but instead "
+            "use shutil.copytree(), which will block until it is complete.  "
+            "fromdir: %s, todir: %s\n" % (fromdir, todir)})
             shutil.copytree(fromdir, todir)
             return defer.succeed(0)
+
+        if not os.path.exists(os.path.dirname(todir)):
+            os.makedirs(os.path.dirname(todir))
+        if os.path.exists(todir):
+            # I don't think this happens, but just in case..
+            log.msg("cp target '%s' already exists -- cp will not do what you think!" % todir)
+
         command = ['cp', '-R', '-P', '-p', fromdir, todir]
         c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         return d
 
     def doPatch(self, res):
         patchlevel, diff = self.patch
         command = [getCommand("patch"), '-p%d' % patchlevel]
         dir = os.path.join(self.builder.basedir, self.workdir)
         # mark the directory so we don't try to update it later
         open(os.path.join(dir, ".buildbot-patched"), "w").write("patched\n")
         # now apply the patch
         c = ShellCommand(self.builder, command, dir,
                          sendRC=False, timeout=self.timeout,
-                         initialStdin=diff)
+                         initialStdin=diff, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         return d
 
 
 class CVS(SourceBase):
     """CVS-specific VC operation. In addition to the arguments handled by
@@ -1465,17 +1634,17 @@ class CVS(SourceBase):
     def start(self):
         if self.login is not None:
             # need to do a 'cvs login' command first
             d = self.builder.basedir
             command = ([self.vcexe, '-d', self.cvsroot] + self.global_options
                        + ['login'])
             c = ShellCommand(self.builder, command, d,
                              sendRC=False, timeout=self.timeout,
-                             initialStdin=self.login+"\n")
+                             initialStdin=self.login+"\n", usePTY=False)
             self.command = c
             d = c.start()
             d.addCallback(self._abandonOnFailure)
             d.addCallback(self._didLogin)
             return d
         else:
             return self._didLogin(None)
 
@@ -1486,17 +1655,17 @@ class CVS(SourceBase):
     def doVCUpdate(self):
         d = os.path.join(self.builder.basedir, self.srcdir)
         command = [self.vcexe, '-z3'] + self.global_options + ['update', '-dP']
         if self.branch:
             command += ['-r', self.branch]
         if self.revision:
             command += ['-D', self.revision]
         c = ShellCommand(self.builder, command, d,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         return c.start()
 
     def doVCFull(self):
         d = self.builder.basedir
         if self.mode == "export":
             verb = "export"
         else:
@@ -1505,17 +1674,17 @@ class CVS(SourceBase):
                    self.global_options +
                    [verb, '-d', self.srcdir])
         if self.branch:
             command += ['-r', self.branch]
         if self.revision:
             command += ['-D', self.revision]
         command += [self.cvsmodule]
         c = ShellCommand(self.builder, command, d,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         return c.start()
 
     def parseGotRevision(self):
         # CVS does not have any kind of revision stamp to speak of. We return
         # the current timestamp as a best-effort guess, but this depends upon
         # the local system having a clock that is
         # reasonably-well-synchronized with the repository.
@@ -1523,60 +1692,74 @@ class CVS(SourceBase):
 
 registerSlaveCommand("cvs", CVS, command_version)
 
 class SVN(SourceBase):
     """Subversion-specific VC operation. In addition to the arguments
     handled by SourceBase, this command reads the following keys:
 
     ['svnurl'] (required): the SVN repository string
+    ['username']    Username passed to the svn command
+    ['password']    Password passed to the svn command
     """
 
     header = "svn operation"
 
     def setup(self, args):
         SourceBase.setup(self, args)
         self.vcexe = getCommand("svn")
         self.svnurl = args['svnurl']
         self.sourcedata = "%s\n" % self.svnurl
 
+        self.extra_args = []
+        if args.has_key('username'):
+            self.extra_args.extend(["--username", args['username']])
+        if args.has_key('password'):
+            self.extra_args.extend(["--password", Obfuscated(args['password'], "XXXX")])
+
     def sourcedirIsUpdateable(self):
         if os.path.exists(os.path.join(self.builder.basedir,
                                        self.srcdir, ".buildbot-patched")):
             return False
         return os.path.isdir(os.path.join(self.builder.basedir,
                                           self.srcdir, ".svn"))
 
     def doVCUpdate(self):
         revision = self.args['revision'] or 'HEAD'
         # update: possible for mode in ('copy', 'update')
         d = os.path.join(self.builder.basedir, self.srcdir)
-        command = [self.vcexe, 'update', '--revision', str(revision),
+        command = [self.vcexe, 'update'] + \
+                    self.extra_args + \
+                    ['--revision', str(revision),
                    '--non-interactive', '--no-auth-cache']
         c = ShellCommand(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         keepStdout=True)
+                         keepStdout=True, usePTY=False)
         self.command = c
         return c.start()
 
     def doVCFull(self):
         revision = self.args['revision'] or 'HEAD'
         d = self.builder.basedir
         if self.mode == "export":
-            command = [self.vcexe, 'export', '--revision', str(revision),
-                       '--non-interactive', '--no-auth-cache',
-                       self.svnurl, self.srcdir]
+            command = [self.vcexe, 'export'] + \
+                        self.extra_args + \
+                        ['--revision', str(revision),
+                        '--non-interactive', '--no-auth-cache',
+                        self.svnurl, self.srcdir]
         else:
             # mode=='clobber', or copy/update on a broken workspace
-            command = [self.vcexe, 'checkout', '--revision', str(revision),
-                       '--non-interactive', '--no-auth-cache',
-                       self.svnurl, self.srcdir]
+            command = [self.vcexe, 'checkout'] + \
+                        self.extra_args + \
+                        ['--revision', str(revision),
+                        '--non-interactive', '--no-auth-cache',
+                        self.svnurl, self.srcdir]
         c = ShellCommand(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         keepStdout=True)
+                         keepStdout=True, usePTY=False)
         self.command = c
         return c.start()
 
     def getSvnVersionCommand(self):
         """
         Get the (shell) command used to determine SVN revision number
         of checked-out code
 
@@ -1591,18 +1774,17 @@ class SVN(SourceBase):
         return [svnversion_command, "."]
 
     def parseGotRevision(self):
         c = ShellCommand(self.builder,
                          self.getSvnVersionCommand(),
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env,
                          sendStdout=False, sendStderr=False, sendRC=False,
-                         keepStdout=True)
-        c.usePTY = False
+                         keepStdout=True, usePTY=False)
         d = c.start()
         def _parse(res):
             r_raw = c.stdout.strip()
             # Extract revision from the version "number" string
             r = r_raw.rstrip('MS')
             r = r.split(':')[-1]
             got_version = None
             try:
@@ -1646,17 +1828,17 @@ class Darcs(SourceBase):
                                           self.srcdir, "_darcs"))
 
     def doVCUpdate(self):
         assert not self.revision
         # update: possible for mode in ('copy', 'update')
         d = os.path.join(self.builder.basedir, self.srcdir)
         command = [self.vcexe, 'pull', '--all', '--verbose']
         c = ShellCommand(self.builder, command, d,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         return c.start()
 
     def doVCFull(self):
         # checkout or export
         d = self.builder.basedir
         command = [self.vcexe, 'get', '--verbose', '--partial',
                    '--repo-name', self.srcdir]
@@ -1667,17 +1849,17 @@ class Darcs(SourceBase):
             f.write(self.revision)
             f.close()
             # tell Darcs to use that context
             command.append('--context')
             command.append(n)
         command.append(self.repourl)
 
         c = ShellCommand(self.builder, command, d,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         if self.revision:
             d.addCallback(self.removeContextFile, n)
         return d
 
     def removeContextFile(self, res, n):
         os.unlink(n)
@@ -1685,18 +1867,17 @@ class Darcs(SourceBase):
 
     def parseGotRevision(self):
         # we use 'darcs context' to find out what we wound up with
         command = [self.vcexe, "changes", "--context"]
         c = ShellCommand(self.builder, command,
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env,
                          sendStdout=False, sendStderr=False, sendRC=False,
-                         keepStdout=True)
-        c.usePTY = False
+                         keepStdout=True, usePTY=False)
         d = c.start()
         d.addCallback(lambda res: c.stdout)
         return d
 
 registerSlaveCommand("darcs", Darcs, command_version)
 
 class Monotone(SourceBase):
     """Monotone-specific VC operation.  In addition to the arguments handled
@@ -1740,31 +1921,31 @@ class Monotone(SourceBase):
         return self._withFreshDb(self._doUpdate)
 
     def _doUpdate(self):
         # update: possible for mode in ('copy', 'update')
         command = [self.monotone, "update",
                    "-r", self.revision,
                    "-b", self.branch]
         c = ShellCommand(self.builder, command, self.full_srcdir,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         return c.start()
 
     def doVCFull(self):
         return self._withFreshDb(self._doFull)
 
     def _doFull(self):
         command = [self.monotone, "--db=" + self.full_db_path,
                    "checkout",
                    "-r", self.revision,
                    "-b", self.branch,
                    self.full_srcdir]
         c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         return c.start()
 
     def _withFreshDb(self, callback):
         self._makefulls()
         # first ensure the db exists and is usable
         if os.path.isfile(self.full_db_path):
             # already exists, so run 'db migrate' in case monotone has been
@@ -1775,29 +1956,29 @@ class Monotone(SourceBase):
             # We'll be doing an initial pull, so up the timeout to 3 hours to
             # make sure it will have time to complete.
             self._pull_timeout = max(self._pull_timeout, 3 * 60 * 60)
             self.sendStatus({"header": "creating database %s\n"
                                        % (self.full_db_path,)})
             command = [self.monotone, "db", "init",
                        "--db=" + self.full_db_path]
         c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         d.addCallback(self._didDbInit)
         d.addCallback(self._didPull, callback)
         return d
 
     def _didDbInit(self, res):
         command = [self.monotone, "--db=" + self.full_db_path,
                    "pull", "--ticker=dot", self.server_addr, self.branch]
         c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=False, timeout=self._pull_timeout)
+                         sendRC=False, timeout=self._pull_timeout, usePTY=False)
         self.sendStatus({"header": "pulling %s from %s\n"
                                    % (self.branch, self.server_addr)})
         self.command = c
         return c.start()
 
     def _didPull(self, res, callback):
         return callback()
 
@@ -1832,58 +2013,95 @@ class Git(SourceBase):
         return self.branch
 
     def sourcedirIsUpdateable(self):
         if os.path.exists(os.path.join(self._fullSrcdir(),
                                        ".buildbot-patched")):
             return False
         return os.path.isdir(os.path.join(self._fullSrcdir(), ".git"))
 
+    def readSourcedata(self):
+        return open(self.sourcedatafile, "r").read()
+
+    # If the repourl matches the sourcedata file, then
+    # we can say that the sourcedata matches.  We can
+    # ignore branch changes, since Git can work with
+    # many branches fetched, and we deal with it properly
+    # in doVCUpdate.
+    def sourcedataMatches(self):
+        try:
+            olddata = self.readSourcedata()
+            if not olddata.startswith(self.repourl+' '):
+                return False
+        except IOError:
+            return False
+        return True
+
     def _didFetch(self, res):
         if self.revision:
             head = self.revision
         else:
             head = 'FETCH_HEAD'
 
         command = ['git', 'reset', '--hard', head]
         c = ShellCommand(self.builder, command, self._fullSrcdir(),
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         return c.start()
 
+    # Update first runs "git clean", removing local changes,
+    # if the branch to be checked out has changed.  This, combined
+    # with the later "git reset" equates clobbering the repo,
+    # but it's much more efficient.
     def doVCUpdate(self):
-        command = ['git', 'fetch', self.repourl, self.branch]
+        try:
+            # Check to see if our branch has changed
+            diffbranch = self.sourcedata != self.readSourcedata()
+        except IOError:
+            diffbranch = False
+        if diffbranch:
+            command = ['git', 'clean', '-f', '-d']
+            c = ShellCommand(self.builder, command, self._fullSrcdir(),
+                             sendRC=False, timeout=self.timeout, usePTY=False)
+            self.command = c
+            d = c.start()
+            d.addCallback(self._abandonOnFailure)
+            d.addCallback(self._didClean)
+            return d
+        return self._didClean(None)
+
+    def _didClean(self, dummy):
+        command = ['git', 'fetch', '-t', self.repourl, self.branch]
         self.sendStatus({"header": "fetching branch %s from %s\n"
                                         % (self.branch, self.repourl)})
         c = ShellCommand(self.builder, command, self._fullSrcdir(),
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         d.addCallback(self._didFetch)
         return d
 
     def _didInit(self, res):
         return self.doVCUpdate()
 
     def doVCFull(self):
         os.mkdir(self._fullSrcdir())
         c = ShellCommand(self.builder, ['git', 'init'], self._fullSrcdir(),
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         d.addCallback(self._didInit)
         return d
 
     def parseGotRevision(self):
         command = ['git', 'rev-parse', 'HEAD']
         c = ShellCommand(self.builder, command, self._fullSrcdir(),
-                         sendRC=False, keepStdout=True)
-        c.usePTY = False
+                         sendRC=False, keepStdout=True, usePTY=False)
         d = c.start()
         def _parse(res):
             hash = c.stdout.strip()
             if len(hash) != 40:
                 return None
             return hash
         d.addCallback(_parse)
         return d
@@ -1931,30 +2149,30 @@ class Arch(SourceBase):
 
     def doVCUpdate(self):
         # update: possible for mode in ('copy', 'update')
         d = os.path.join(self.builder.basedir, self.srcdir)
         command = [self.vcexe, 'replay']
         if self.revision:
             command.append(self.revision)
         c = ShellCommand(self.builder, command, d,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         return c.start()
 
     def doVCFull(self):
         # to do a checkout, we must first "register" the archive by giving
         # the URL to tla, which will go to the repository at that URL and
         # figure out the archive name. tla will tell you the archive name
         # when it is done, and all further actions must refer to this name.
 
         command = [self.vcexe, 'register-archive', '--force', self.url]
         c = ShellCommand(self.builder, command, self.builder.basedir,
                          sendRC=False, keepStdout=True,
-                         timeout=self.timeout)
+                         timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         d.addCallback(self._didRegister, c)
         return d
 
     def _didRegister(self, res, c):
         # find out what tla thinks the archive name is. If the user told us
@@ -1977,45 +2195,44 @@ class Arch(SourceBase):
     def _doGet(self):
         ver = self.version
         if self.revision:
             ver += "--%s" % self.revision
         command = [self.vcexe, 'get', '--archive', self.archive,
                    '--no-pristine',
                    ver, self.srcdir]
         c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         if self.buildconfig:
             d.addCallback(self._didGet)
         return d
 
     def _didGet(self, res):
         d = os.path.join(self.builder.basedir, self.srcdir)
         command = [self.vcexe, 'build-config', self.buildconfig]
         c = ShellCommand(self.builder, command, d,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         return d
 
     def parseGotRevision(self):
         # using code from tryclient.TlaExtractor
         # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION
         # 'tla logs' gives us REVISION
         command = [self.vcexe, "logs", "--full", "--reverse"]
         c = ShellCommand(self.builder, command,
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env,
                          sendStdout=False, sendStderr=False, sendRC=False,
-                         keepStdout=True)
-        c.usePTY = False
+                         keepStdout=True, usePTY=False)
         d = c.start()
         def _parse(res):
             tid = c.stdout.split("\n")[0].strip()
             slash = tid.index("/")
             dd = tid.rindex("--")
             #branch = tid[slash+1:dd]
             baserev = tid[dd+2:]
             return baserev
@@ -2050,33 +2267,32 @@ class Bazaar(Arch):
         # baz prefers ARCHIVE/VERSION. This will work even if
         # my-default-archive is not set.
         ver = self.archive + "/" + self.version
         if self.revision:
             ver += "--%s" % self.revision
         command = [self.vcexe, 'get', '--no-pristine',
                    ver, self.srcdir]
         c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         if self.buildconfig:
             d.addCallback(self._didGet)
         return d
 
     def parseGotRevision(self):
         # using code from tryclient.BazExtractor
         command = [self.vcexe, "tree-id"]
         c = ShellCommand(self.builder, command,
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env,
                          sendStdout=False, sendStderr=False, sendRC=False,
-                         keepStdout=True)
-        c.usePTY = False
+                         keepStdout=True, usePTY=False)
         d = c.start()
         def _parse(res):
             tid = c.stdout.strip()
             slash = tid.index("/")
             dd = tid.rindex("--")
             #branch = tid[slash+1:dd]
             baserev = tid[dd+2:]
             return baserev
@@ -2113,17 +2329,17 @@ class Bzr(SourceBase):
                                           self.srcdir, ".bzr"))
 
     def doVCUpdate(self):
         assert not self.revision
         # update: possible for mode in ('copy', 'update')
         srcdir = os.path.join(self.builder.basedir, self.srcdir)
         command = [self.vcexe, 'update']
         c = ShellCommand(self.builder, command, srcdir,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         return c.start()
 
     def doVCFull(self):
         # checkout or export
         d = self.builder.basedir
         if self.mode == "export":
             # exporting in bzr requires a separate directory
@@ -2141,38 +2357,38 @@ class Bzr(SourceBase):
         command = [self.vcexe, 'checkout']
         if self.revision:
             command.append('--revision')
             command.append(str(self.revision))
         command.append(self.repourl)
         command.append(self.srcdir)
 
         c = ShellCommand(self.builder, command, d,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         return d
 
     def doVCExport(self):
         tmpdir = os.path.join(self.builder.basedir, "export-temp")
         srcdir = os.path.join(self.builder.basedir, self.srcdir)
         command = [self.vcexe, 'checkout', '--lightweight']
         if self.revision:
             command.append('--revision')
             command.append(str(self.revision))
         command.append(self.repourl)
         command.append(tmpdir)
         c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         d = c.start()
         def _export(res):
             command = [self.vcexe, 'export', srcdir]
             c = ShellCommand(self.builder, command, tmpdir,
-                             sendRC=False, timeout=self.timeout)
+                             sendRC=False, timeout=self.timeout, usePTY=False)
             self.command = c
             return c.start()
         d.addCallback(_export)
         return d
 
     def get_revision_number(self, out):
         # it feels like 'bzr revno' sometimes gives different results than
         # the 'revno:' line from 'bzr version-info', and the one from
@@ -2186,18 +2402,17 @@ class Bzr(SourceBase):
         raise ValueError("unable to find revno: in bzr output: '%s'" % out)
 
     def parseGotRevision(self):
         command = [self.vcexe, "version-info"]
         c = ShellCommand(self.builder, command,
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env,
                          sendStdout=False, sendStderr=False, sendRC=False,
-                         keepStdout=True)
-        c.usePTY = False
+                         keepStdout=True, usePTY=False)
         d = c.start()
         def _parse(res):
             try:
                 return self.get_revision_number(c.stdout)
             except ValueError:
                 msg =("Bzr.parseGotRevision unable to parse output "
                       "of bzr version-info: '%s'" % c.stdout.strip())
                 log.msg(msg)
@@ -2233,144 +2448,197 @@ class Mercurial(SourceBase):
         # full checkout. TODO: I think 'hg pull' plus 'hg update' might work
         if self.revision:
             return False
         return os.path.isdir(os.path.join(self.builder.basedir,
                                           self.srcdir, ".hg"))
 
     def doVCUpdate(self):
         d = os.path.join(self.builder.basedir, self.srcdir)
-        command = [self.vcexe, 'pull', '--update', '--verbose']
-        if self.args['revision']:
-            command.extend(['--rev', self.args['revision']])
+        command = [self.vcexe, 'pull', '--verbose', self.repourl]
         c = ShellCommand(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         keepStdout=True)
+                         keepStdout=True, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._handleEmptyUpdate)
+        d.addCallback(self._update)
         return d
 
     def _handleEmptyUpdate(self, res):
         if type(res) is int and res == 1:
             if self.command.stdout.find("no changes found") != -1:
                 # 'hg pull', when it doesn't have anything to do, exits with
                 # rc=1, and there appears to be no way to shut this off. It
                 # emits a distinctive message to stdout, though. So catch
                 # this and pretend that it completed successfully.
                 return 0
         return res
 
     def doVCFull(self):
-        newdir = os.path.join(self.builder.basedir, self.srcdir)
-        command = [self.vcexe, 'clone']
-        if self.args['revision']:
-            command.extend(['--rev', self.args['revision']])
-        command.extend([self.repourl, newdir])
+        d = os.path.join(self.builder.basedir, self.srcdir)
+        command = [self.vcexe, 'init', d]
         c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=False, keepStdout=True, keepStderr=True,
-                         timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
-        d = c.start()
-        d.addCallback(self._maybeFallback, c)
-        return d
-
-    def _maybeFallback(self, res, c):
-        # to do 'hg clone -r REV' (i.e. to check out a specific revision)
-        # from a remote (HTTP) repository, both the client and the server
-        # need to be hg-0.9.2 or newer. If this caused a checkout failure, we
-        # fall back to doing a checkout of HEAD (spelled 'tip' in hg
-        # parlance) and then 'hg update' *backwards* to the desired revision.
-        if res == 0:
+        cmd1 = c.start()
+
+        def _vcupdate(res):
+            return self.doVCUpdate()
+        
+        cmd1.addCallback(_vcupdate)
+        return cmd1
+
+    def _update(self, res):
+        if res != 0:
             return res
-
-        errmsgs = [
-            # hg-0.6 didn't even have the 'clone' command
-            # hg-0.7
-            "hg clone: option --rev not recognized",
-            # hg-0.8, 0.8.1, 0.9
-            "abort: clone -r not supported yet for remote repositories.",
-            # hg-0.9.1
-            ("abort: clone by revision not supported yet for "
-             "remote repositories"),
-            # hg-0.9.2 and later say this when the other end is too old
-            ("abort: src repository does not support revision lookup "
-             "and so doesn't support clone by revision"),
-            ]
-
-        fallback_is_useful = False
-        for errmsg in errmsgs:
-            # the error message might be in stdout if we're using PTYs, which
-            # merge stdout and stderr.
-            if errmsg in c.stdout or errmsg in c.stderr:
-                fallback_is_useful = True
-                break
-        if not fallback_is_useful:
-            return res # must be some other error
-
-        # ok, do the fallback
-        newdir = os.path.join(self.builder.basedir, self.srcdir)
-        command = [self.vcexe, 'clone']
-        command.extend([self.repourl, newdir])
-        c = ShellCommand(self.builder, command, self.builder.basedir,
-                         sendRC=False, timeout=self.timeout)
-        self.command = c
-        d = c.start()
-        d.addCallback(self._abandonOnFailure)
-        d.addCallback(self._updateToDesiredRevision)
-        return d
-
-    def _updateToDesiredRevision(self, res):
-        assert self.args['revision']
-        newdir = os.path.join(self.builder.basedir, self.srcdir)
-        # hg-0.9.1 and earlier (which need this fallback) also want to see
-        # 'hg update REV' instead of 'hg update --rev REV'. Note that this is
-        # the only place we use 'hg update', since what most VC tools mean
-        # by, say, 'cvs update' is expressed as 'hg pull --update' instead.
-        command = [self.vcexe, 'update', self.args['revision']]
-        c = ShellCommand(self.builder, command, newdir,
-                         sendRC=False, timeout=self.timeout)
-        return c.start()
+                
+        # compare current branch to update
+        self.update_branch = self.args.get('branch',  'default')
+
+        d = os.path.join(self.builder.basedir, self.srcdir)
+        parentscmd = [self.vcexe, 'identify', '--num', '--branch']
+        cmd = ShellCommand(self.builder, parentscmd, d,
+                           sendStdout=False, sendStderr=False,
+                           keepStdout=True, keepStderr=True, usePTY=False)
+        
+        def _parse(res):
+            if res != 0:
+                msg = "'hg identify' failed: %s\n%s" % (cmd.stdout, cmd.stderr)
+                self.sendStatus({'header': msg + "\n"})
+                log.msg(msg)
+                return res
+            
+            log.msg('Output: %s' % cmd.stdout)
+                        
+            match = re.search(r'^(.+) (.+)$', cmd.stdout)
+            assert match
+            
+            rev = match.group(1)
+            current_branch = match.group(2)
+            
+            if rev == '-1':
+                msg = "Fresh hg repo, don't worry about branch"
+                log.msg(msg)
+                        
+            elif self.update_branch != current_branch:
+                msg = "Working dir is on branch '%s' and build needs '%s'. Clobbering." % (current_branch, self.update_branch)
+                self.sendStatus({'header': msg + "\n"})
+                log.msg(msg)
+                
+                def _vcfull(res):
+                    return self.doVCFull()
+                
+                d = self.doClobber(None, self.srcdir)                
+                d.addCallback(_vcfull)
+                return d
+                
+            else:
+                msg = "Working dir on same branch as build (%s)." % (current_branch)
+                log.msg(msg)
+                        
+            return 0            
+        
+        c = cmd.start()                
+        c.addCallback(_parse)
+        c.addCallback(self._update2)
+        return c
+        
+    def _update2(self, res):                        
+        d = os.path.join(self.builder.basedir, self.srcdir)
+
+        updatecmd=[self.vcexe, 'update', '--clean', '--repository', d]
+        if self.args.get('revision'):
+            updatecmd.extend(['--rev', self.args['revision']])
+        else:
+            updatecmd.extend(['--rev', self.args.get('branch',  'default')])
+        self.command = ShellCommand(self.builder, updatecmd,
+            self.builder.basedir, sendRC=False,
+            timeout=self.timeout, usePTY=False)
+        return self.command.start()
 
     def parseGotRevision(self):
         # we use 'hg identify' to find out what we wound up with
         command = [self.vcexe, "identify"]
         c = ShellCommand(self.builder, command,
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env,
                          sendStdout=False, sendStderr=False, sendRC=False,
-                         keepStdout=True)
+                         keepStdout=True, usePTY=False)
         d = c.start()
         def _parse(res):
             m = re.search(r'^(\w+)', c.stdout)
             return m.group(1)
         d.addCallback(_parse)
         return d
 
 registerSlaveCommand("hg", Mercurial, command_version)
 
 
-class P4(SourceBase):
+class P4Base(SourceBase):
+    """Base class for P4 source-updaters
+
+    ['p4port'] (required): host:port for server to access
+    ['p4user'] (optional): user to use for access
+    ['p4passwd'] (optional): passwd to try for the user
+    ['p4client'] (optional): client spec to use
+    """
+    def setup(self, args):
+        SourceBase.setup(self, args)
+        self.p4port = args['p4port']
+        self.p4client = args['p4client']
+        self.p4user = args['p4user']
+        self.p4passwd = args['p4passwd']
+
+    def parseGotRevision(self):
+        # Executes a p4 command that will give us the latest changelist number
+        # of any file under the current (or default) client:
+        command = ['p4']
+        if self.p4port:
+            command.extend(['-p', self.p4port])
+        if self.p4user:
+            command.extend(['-u', self.p4user])
+        if self.p4passwd:
+            command.extend(['-P', self.p4passwd])
+        if self.p4client:
+            command.extend(['-c', self.p4client])
+        command.extend(['changes', '-m', '1', '#have'])
+        c = ShellCommand(self.builder, command, self.builder.basedir,
+                         environ=self.env, timeout=self.timeout,
+                         sendStdout=True, sendStderr=False, sendRC=False,
+                         keepStdout=True, usePTY=False)
+        self.command = c
+        d = c.start()
+
+        def _parse(res):
+            # 'p4 -c clien-name change -m 1 "#have"' will produce an output like:
+            # "Change 28147 on 2008/04/07 by p4user@hostname..."
+            # The number after "Change" is the one we want.
+            m = re.match('Change\s+(\d+)\s+', c.stdout)
+            if m:
+                return m.group(1)
+            return None
+        d.addCallback(_parse)
+        return d
+
+
+class P4(P4Base):
     """A P4 source-updater.
 
     ['p4port'] (required): host:port for server to access
     ['p4user'] (optional): user to use for access
     ['p4passwd'] (optional): passwd to try for the user
     ['p4client'] (optional): client spec to use
     ['p4extra_views'] (optional): additional client views to use
     """
 
     header = "p4"
 
     def setup(self, args):
-        SourceBase.setup(self, args)
-        self.p4port = args['p4port']
-        self.p4client = args['p4client']
-        self.p4user = args['p4user']
-        self.p4passwd = args['p4passwd']
+        P4Base.setup(self, args)
         self.p4base = args['p4base']
         self.p4extra_views = args['p4extra_views']
         self.p4mode = args['mode']
         self.p4branch = args['branch']
 
         self.sourcedata = str([
             # Perforce server.
             self.p4port,
@@ -2417,17 +2685,17 @@ class P4(SourceBase):
         command.extend(['sync'])
         if force:
             command.extend(['-f'])
         if self.revision:
             command.extend(['@' + str(self.revision)])
         env = {}
         c = ShellCommand(self.builder, command, self.builder.basedir,
                          environ=env, sendRC=False, timeout=self.timeout,
-                         keepStdout=True)
+                         keepStdout=True, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         return d
 
 
     def doVCFull(self):
         env = {}
@@ -2454,46 +2722,42 @@ class P4(SourceBase):
         if self.p4user:
             command.extend(['-u', self.p4user])
         if self.p4passwd:
             command.extend(['-P', self.p4passwd])
         command.extend(['client', '-i'])
         log.msg(client_spec)
         c = ShellCommand(self.builder, command, self.builder.basedir,
                          environ=env, sendRC=False, timeout=self.timeout,
-                         initialStdin=client_spec)
+                         initialStdin=client_spec, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
         d.addCallback(lambda _: self._doP4Sync(force=True))
         return d
 
 registerSlaveCommand("p4", P4, command_version)
 
 
-class P4Sync(SourceBase):
+class P4Sync(P4Base):
     """A partial P4 source-updater. Requires manual setup of a per-slave P4
     environment. The only thing which comes from the master is P4PORT.
     'mode' is required to be 'copy'.
 
     ['p4port'] (required): host:port for server to access
     ['p4user'] (optional): user to use for access
     ['p4passwd'] (optional): passwd to try for the user
     ['p4client'] (optional): client spec to use
     """
 
     header = "p4 sync"
 
     def setup(self, args):
-        SourceBase.setup(self, args)
+        P4Base.setup(self, args)
         self.vcexe = getCommand("p4")
-        self.p4port = args['p4port']
-        self.p4user = args['p4user']
-        self.p4passwd = args['p4passwd']
-        self.p4client = args['p4client']
 
     def sourcedirIsUpdateable(self):
         return True
 
     def _doVC(self, force):
         d = os.path.join(self.builder.basedir, self.srcdir)
         command = [self.vcexe]
         if self.p4port:
@@ -2506,17 +2770,17 @@ class P4Sync(SourceBase):
             command.extend(['-c', self.p4client])
         command.extend(['sync'])
         if force:
             command.extend(['-f'])
         if self.revision:
             command.extend(['@' + self.revision])
         env = {}
         c = ShellCommand(self.builder, command, d, environ=env,
-                         sendRC=False, timeout=self.timeout)
+                         sendRC=False, timeout=self.timeout, usePTY=False)
         self.command = c
         return c.start()
 
     def doVCUpdate(self):
         return self._doVC(force=False)
 
     def doVCFull(self):
         return self._doVC(force=True)
--- a/buildbot/status/base.py
+++ b/buildbot/status/base.py
@@ -3,16 +3,19 @@ from zope.interface import implements
 from twisted.application import service
 
 from buildbot.interfaces import IStatusReceiver
 from buildbot import util, pbutil
 
 class StatusReceiver:
     implements(IStatusReceiver)
 
+    def requestSubmitted(self, request):
+        pass
+
     def buildsetSubmitted(self, buildset):
         pass
 
     def builderAdded(self, builderName, builder):
         pass
 
     def builderChangedState(self, builderName, state):
         pass
@@ -21,16 +24,22 @@ class StatusReceiver:
         pass
 
     def buildETAUpdate(self, build, ETA):
         pass
 
     def stepStarted(self, build, step):
         pass
 
+    def stepTextChanged(self, build, step, text):
+        pass
+
+    def stepText2Changed(self, build, step, text2):
+        pass
+
     def stepETAUpdate(self, build, step, ETA, expectations):
         pass
 
     def logStarted(self, build, step, log):
         pass
 
     def logChunk(self, build, step, log, channel, text):
         pass
--- a/buildbot/status/builder.py
+++ b/buildbot/status/builder.py
@@ -1,20 +1,21 @@
 # -*- test-case-name: buildbot.test.test_status -*-
 
 from zope.interface import implements
 from twisted.python import log
 from twisted.persisted import styles
-from twisted.internet import reactor, defer
+from twisted.internet import reactor, defer, threads
 from twisted.protocols import basic
 from buildbot.process.properties import Properties
 
 import os, shutil, sys, re, urllib, itertools
 from cPickle import load, dump
 from cStringIO import StringIO
+from bz2 import BZ2File
 
 # sibling imports
 from buildbot import interfaces, util, sourcestamp
 
 SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION = range(5)
 Results = ["success", "warnings", "failure", "skipped", "exception"]
 
 
@@ -179,16 +180,32 @@ class LogFileProducer:
 
     def logfileFinished(self, logfile):
         self.done()
         if self.consumer:
             self.consumer.unregisterProducer()
             self.consumer.finish()
             self.consumer = None
 
+def _tryremove(filename, timeout, retries):
+    """Try to remove a file, and if failed, try again in timeout.
+    Increases the timeout by a factor of 4, and only keeps trying for
+    another retries-amount of times.
+
+    """
+    try:
+        os.unlink(filename)
+    except OSError:
+        if retries > 0:
+            reactor.callLater(timeout, _tryremove, filename, timeout * 4, 
+                              retries - 1)
+        else:
+            log.msg("giving up on removing %s after over %d seconds" %
+                    (filename, timeout))
+
 class LogFile:
     """A LogFile keeps all of its contents on disk, in a non-pickle format to
     which new entries can easily be appended. The file on disk has a name
     like 12-log-compile-output, under the Builder's directory. The actual
     filename is generated (before the LogFile is created) by
     L{BuildStatus.generateLogfileName}.
 
     Old LogFile pickles (which kept their contents in .entries) must be
@@ -232,17 +249,18 @@ class LogFile:
         self.runEntries = []
         self.watchers = []
         self.finishedWatchers = []
 
     def getFilename(self):
         return os.path.join(self.step.build.builder.basedir, self.filename)
 
     def hasContents(self):
-        return os.path.exists(self.getFilename())
+        return os.path.exists(self.getFilename() + '.bz2') or \
+            os.path.exists(self.getFilename())
 
     def getName(self):
         return self.name
 
     def getStep(self):
         return self.step
 
     def isFinished(self):
@@ -256,16 +274,21 @@ class LogFile:
         return d
 
     def getFile(self):
         if self.openfile:
             # this is the filehandle we're using to write to the log, so
             # don't close it!
             return self.openfile
         # otherwise they get their own read-only handle
+        # try a compressed log first
+        try:
+            return BZ2File(self.getFilename() + ".bz2", "r")
+        except IOError:
+            pass
         return open(self.getFilename(), "r")
 
     def getText(self):
         # this produces one ginormous string
         return "".join(self.getChunks([STDOUT, STDERR], onlyText=True))
 
     def getTextWithHeaders(self):
         return "".join(self.getChunks(onlyText=True))
@@ -402,25 +425,61 @@ class LogFile:
     def finish(self):
         self.merge()
         if self.openfile:
             # we don't do an explicit close, because there might be readers
             # shareing the filehandle. As soon as they stop reading, the
             # filehandle will be released and automatically closed. We will
             # do a sync, however, to make sure the log gets saved in case of
             # a crash.
+            self.openfile.flush()
             os.fsync(self.openfile.fileno())
             del self.openfile
         self.finished = True
         watchers = self.finishedWatchers
         self.finishedWatchers = []
         for w in watchers:
             w.callback(self)
         self.watchers = []
 
+
+    def compressLog(self):
+        compressed = self.getFilename() + ".bz2.tmp"
+        d = threads.deferToThread(self._compressLog, compressed)
+        d.addCallback(self._renameCompressedLog, compressed)
+        d.addErrback(self._cleanupFailedCompress, compressed)
+        return d
+
+    def _compressLog(self, compressed):
+        infile = self.getFile()
+        cf = BZ2File(compressed, 'w')
+        bufsize = 1024*1024
+        while True:
+            buf = infile.read(bufsize)
+            cf.write(buf)
+            if len(buf) < bufsize:
+                break
+        cf.close()
+    def _renameCompressedLog(self, rv, compressed):
+        filename = self.getFilename() + '.bz2'
+        if sys.platform == 'win32':
+            # windows cannot rename a file on top of an existing one, so
+            # fall back to delete-first. There are ways this can fail and
+            # lose the builder's history, so we avoid using it in the
+            # general (non-windows) case
+            if os.path.exists(filename):
+                os.unlink(filename)
+        os.rename(compressed, filename)
+        _tryremove(self.getFilename(), 1, 5)
+    def _cleanupFailedCompress(self, failure, compressed):
+        log.msg("failed to compress %s" % self.getFilename())
+        if os.path.exists(compressed):
+            _tryremove(compressed, 1, 5)
+        failure.trap() # reraise the failure
+
     # persistence stuff
     def __getstate__(self):
         d = self.__dict__.copy()
         del d['step'] # filled in upon unpickling
         del d['watchers']
         del d['finishedWatchers']
         d['entries'] = [] # let 0.6.4 tolerate the saved log. TODO: really?
         if d.has_key('finished'):
@@ -497,25 +556,22 @@ class HTMLLogFile:
 
 
 class Event:
     implements(interfaces.IStatusEvent)
 
     started = None
     finished = None
     text = []
-    color = None
 
     # IStatusEvent methods
     def getTimes(self):
         return (self.started, self.finished)
     def getText(self):
         return self.text
-    def getColor(self):
-        return self.color
     def getLogs(self):
         return []
 
     def finish(self):
         self.finished = util.now()
 
 class TestResult:
     implements(interfaces.ITestResult)
@@ -609,16 +665,17 @@ class BuildSetStatus:
 class BuildRequestStatus:
     implements(interfaces.IBuildRequestStatus)
 
     def __init__(self, source, builderName):
         self.source = source
         self.builderName = builderName
         self.builds = [] # list of BuildStatus objects
         self.observers = []
+        self.submittedAt = None
 
     def buildStarted(self, build):
         self.builds.append(build)
         for o in self.observers[:]:
             o(build)
 
     # methods called by our clients
     def getSourceStamp(self):
@@ -630,32 +687,32 @@ class BuildRequestStatus:
 
     def subscribe(self, observer):
         self.observers.append(observer)
         for b in self.builds:
             observer(b)
     def unsubscribe(self, observer):
         self.observers.remove(observer)
 
+    def getSubmitTime(self):
+        return self.submittedAt
+    def setSubmitTime(self, t):
+        self.submittedAt = t
+
 
 class BuildStepStatus(styles.Versioned):
     """
     I represent a collection of output status for a
     L{buildbot.process.step.BuildStep}.
 
     Statistics contain any information gleaned from a step that is
     not in the form of a logfile.  As an example, steps that run
     tests might gather statistics about the number of passed, failed,
     or skipped tests.
 
-    @type color: string
-    @cvar color: color that this step feels best represents its
-                 current mood. yellow,green,red,orange are the
-                 most likely choices, although purple indicates
-                 an exception
     @type progress: L{buildbot.status.progress.StepProgress}
     @cvar progress: tracks ETA for the step
     @type text: list of strings
     @cvar text: list of short texts that describe the command and its status
     @type text2: list of strings
     @cvar text2: list of short texts added to the overall build description
     @type logs: dict of string -> L{buildbot.status.builder.LogFile}
     @ivar logs: logs of steps
@@ -666,17 +723,16 @@ class BuildStepStatus(styles.Versioned):
     # corresponding BuildStep has started.
     implements(interfaces.IBuildStepStatus, interfaces.IStatusEvent)
     persistenceVersion = 2
 
     started = None
     finished = None
     progress = None
     text = []
-    color = None
     results = (None, [])
     text2 = []
     watchers = []
     updates = {}
     finishedWatchers = []
     statistics = {}
 
     def __init__(self, parent):
@@ -746,22 +802,16 @@ class BuildStepStatus(styles.Versioned):
 
     def getText(self):
         """Returns a list of strings which describe the step. These are
         intended to be displayed in a narrow column. If more space is
         available, the caller should join them together with spaces before
         presenting them to the user."""
         return self.text
 
-    def getColor(self):
-        """Returns a single string with the color that should be used to
-        display this step. 'green', 'orange', 'red', 'yellow' and 'purple'
-        are the most likely ones."""
-        return self.color
-
     def getResults(self):
         """Return a tuple describing the results of the step.
         'result' is one of the constants in L{buildbot.status.builder}:
         SUCCESS, WARNINGS, FAILURE, or SKIPPED.
         'strings' is an optional list of strings that the step wants to
         append to the overall build's results. These strings are usually
         more terse than the ones returned by getText(): in particular,
         successful Steps do not usually contribute any text to the
@@ -810,16 +860,19 @@ class BuildStepStatus(styles.Versioned):
             del self.updates[receiver]
 
 
     # methods to be invoked by the BuildStep
 
     def setName(self, stepname):
         self.name = stepname
 
+    def setColor(self, color):
+        log.msg("BuildStepStatus.setColor is no longer supported -- ignoring color %s" % (color,))
+
     def setProgress(self, stepprogress):
         self.progress = stepprogress
 
     def stepStarted(self):
         self.started = util.now()
         if self.build:
             self.build.stepStarted(self)
 
@@ -853,44 +906,56 @@ class BuildStepStatus(styles.Versioned):
 
     def logFinished(self, log):
         for w in self.watchers:
             w.logFinished(self.build, self, log)
 
     def addURL(self, name, url):
         self.urls[name] = url
 
-    def setColor(self, color):
-        self.color = color
     def setText(self, text):
         self.text = text
+        for w in self.watchers:
+            w.stepTextChanged(self.build, self, text)
     def setText2(self, text):
         self.text2 = text
+        for w in self.watchers:
+            w.stepText2Changed(self.build, self, text)
 
     def setStatistic(self, name, value):
         """Set the given statistic.  Usually called by subclasses.
         """
         self.statistics[name] = value
 
     def stepFinished(self, results):
         self.finished = util.now()
         self.results = results
+        cld = [] # deferreds for log compression
+        logCompressionLimit = self.build.builder.logCompressionLimit
         for loog in self.logs:
             if not loog.isFinished():
                 loog.finish()
+            # if log compression is on, and it's a real LogFile,
+            # HTMLLogFiles aren't files
+            if logCompressionLimit is not False and \
+                    isinstance(loog, LogFile):
+                if os.path.getsize(loog.getFilename()) > logCompressionLimit:
+                    cld.append(loog.compressLog())
 
         for r in self.updates.keys():
             if self.updates[r] is not None:
                 self.updates[r].cancel()
                 del self.updates[r]
 
         watchers = self.finishedWatchers
         self.finishedWatchers = []
         for w in watchers:
             w.callback(self)
+        if cld:
+            return defer.DeferredList(cld)
 
     # persistence
 
     def __getstate__(self):
         d = styles.Versioned.__getstate__(self)
         del d['build'] # filled in when loading
         if d.has_key('progress'):
             del d['progress']
@@ -917,22 +982,22 @@ class BuildStepStatus(styles.Versioned):
 class BuildStatus(styles.Versioned):
     implements(interfaces.IBuildStatus, interfaces.IStatusEvent)
     persistenceVersion = 3
 
     source = None
     reason = None
     changes = []
     blamelist = []
+    requests = []
     progress = None
     started = None
     finished = None
     currentStep = None
     text = []
-    color = None
     results = None
     slavename = "???"
 
     # these lists/dicts are defined here so that unserialized instances have
     # (empty) values. They are set in __init__ to new objects to make sure
     # each instance gets its own copy.
     watchers = []
     updates = {}
@@ -948,16 +1013,17 @@ class BuildStatus(styles.Versioned):
         self.builder = parent
         self.number = number
         self.watchers = []
         self.updates = {}
         self.finishedWatchers = []
         self.steps = []
         self.testResults = {}
         self.properties = Properties()
+        self.requests = []
 
     # IBuildStatus
 
     def getBuilder(self):
         """
         @rtype: L{BuilderStatus}
         """
         return self.builder
@@ -982,22 +1048,25 @@ class BuildStatus(styles.Versioned):
         return self.source.getAbsoluteSourceStamp(self.properties['got_revision'])
 
     def getReason(self):
         return self.reason
 
     def getChanges(self):
         return self.changes
 
+    def getRequests(self):
+        return self.requests
+
     def getResponsibleUsers(self):
         return self.blamelist
 
     def getInterestedUsers(self):
         # TODO: the Builder should add others: sheriffs, domain-owners
-        return self.blamelist
+        return self.blamelist + self.properties.getProperty('owners', [])
 
     def getSteps(self):
         """Return a list of IBuildStepStatus objects. For invariant builds
         (those which always use the same set of Steps), this should be the
         complete list, however some of the steps may not have started yet
         (step.getTimes()[0] will be None). For variant builds, this may not
         be complete (asking again later may give you more of them)."""
         return self.steps
@@ -1054,19 +1123,16 @@ class BuildStatus(styles.Versioned):
 
     def getText(self):
         text = []
         text.extend(self.text)
         for s in self.steps:
             text.extend(s.text2)
         return text
 
-    def getColor(self):
-        return self.color
-
     def getResults(self):
         return self.results
 
     def getSlavename(self):
         return self.slavename
 
     def getTestResults(self):
         return self.testResults
@@ -1127,16 +1193,19 @@ class BuildStatus(styles.Versioned):
 
     def addTestResult(self, result):
         self.testResults[result.getName()] = result
 
     def setSourceStamp(self, sourceStamp):
         self.source = sourceStamp
         self.changes = self.source.changes
 
+    def setRequests(self, requests):
+        self.requests = requests
+
     def setReason(self, reason):
         self.reason = reason
     def setBlamelist(self, blamelist):
         self.blamelist = blamelist
     def setProgress(self, progress):
         self.progress = progress
 
     def buildStarted(self, build):
@@ -1149,18 +1218,16 @@ class BuildStatus(styles.Versioned):
         self.builder.buildStarted(self)
 
     def setSlavename(self, slavename):
         self.slavename = slavename
 
     def setText(self, text):
         assert isinstance(text, (list, tuple))
         self.text = text
-    def setColor(self, color):
-        self.color = color
     def setResults(self, results):
         self.results = results
 
     def buildFinished(self):
         self.currentStep = None
         self.finished = util.now()
 
         for r in self.updates.keys():
@@ -1244,16 +1311,17 @@ class BuildStatus(styles.Versioned):
             d['finished'] = True
             # TODO: push an "interrupted" step so it is clear that the build
             # was interrupted. The builder will have a 'shutdown' event, but
             # someone looking at just this build will be confused as to why
             # the last log is truncated.
         del d['builder'] # filled in by our parent when loading
         del d['watchers']
         del d['updates']
+        del d['requests']
         del d['finishedWatchers']
         return d
 
     def __setstate__(self, d):
         styles.Versioned.__setstate__(self, d)
         # self.builder must be filled in by our parent when loading
         for step in self.steps:
             step.build = self
@@ -1365,16 +1433,17 @@ class BuilderStatus(styles.Versioned):
         self.lastBuildStatus = None
         #self.currentBig = None
         #self.currentSmall = None
         self.currentBuilds = []
         self.pendingBuilds = []
         self.nextBuild = None
         self.watchers = []
         self.buildCache = [] # TODO: age builds out of the cache
+        self.logCompressionLimit = False # default to no compression for tests
 
     # persistence
 
     def __getstate__(self):
         # when saving, don't record transient stuff like what builds are
         # currently running, because they won't be there when we start back
         # up. Nor do we save self.watchers, nor anything that gets set by our
         # parent like .basedir and .status
@@ -1420,16 +1489,19 @@ class BuilderStatus(styles.Versioned):
         existing_builds = [int(f)
                            for f in os.listdir(self.basedir)
                            if re.match("^\d+$", f)]
         if existing_builds:
             self.nextBuildNumber = max(existing_builds) + 1
         else:
             self.nextBuildNumber = 0
 
+    def setLogCompressionLimit(self, lowerLimit):
+        self.logCompressionLimit = lowerLimit
+
     def saveYourself(self):
         for b in self.buildCache:
             if not b.isFinished:
                 # interrupted build, need to save it anyway.
                 # BuildStatus.saveYourself will mark it as interrupted.
                 b.saveYourself()
         filename = os.path.join(self.basedir, "builder")
         tmpfilename = filename + ".tmp"
@@ -1599,34 +1671,32 @@ class BuilderStatus(styles.Versioned):
     def unsubscribe(self, receiver):
         self.watchers.remove(receiver)
 
     ## Builder interface (methods called by the Builder which feeds us)
 
     def setSlavenames(self, names):
         self.slavenames = names
 
-    def addEvent(self, text=[], color=None):
+    def addEvent(self, text=[]):
         # this adds a duration event. When it is done, the user should call
-        # e.finish(). They can also mangle it by modifying .text and .color
+        # e.finish(). They can also mangle it by modifying .text
         e = Event()
         e.started = util.now()
         e.text = text
-        e.color = color
         self.events.append(e)
         return e # they are free to mangle it further
 
-    def addPointEvent(self, text=[], color=None):
+    def addPointEvent(self, text=[]):
         # this adds a point event, one which occurs as a single atomic
         # instant of time.
         e = Event()
         e.started = util.now()
         e.finished = 0
         e.text = text
-        e.color = color
         self.events.append(e)
         return e # for consistency, but they really shouldn't touch it
 
     def setBigState(self, state):
         needToUpdate = state != self.currentBigState
         self.currentBigState = state
         if needToUpdate:
             self.publishState()
@@ -1634,17 +1704,21 @@ class BuilderStatus(styles.Versioned):
     def publishState(self, target=None):
         state = self.currentBigState
 
         if target is not None:
             # unicast
             target.builderChangedState(self.name, state)
             return
         for w in self.watchers:
-            w.builderChangedState(self.name, state)
+            try:
+                w.builderChangedState(self.name, state)
+            except:
+                log.msg("Exception caught publishing state to %r" % w)
+                log.err()
 
     def newBuild(self):
         """The Builder has decided to start a build, but the Build object is
         not yet ready to report status (it has not finished creating the
         Steps). Create a BuildStatus object that it can use."""
         number = self.nextBuildNumber
         self.nextBuildNumber += 1
         # TODO: self.saveYourself(), to make sure we don't forget about the
@@ -1652,16 +1726,19 @@ class BuilderStatus(styles.Versioned):
         # as it was before we switch to determineNextBuildNumber, but I think
         # it may still be useful to have the new build save itself.
         s = BuildStatus(self, number)
         s.waitUntilFinished().addCallback(self._buildFinished)
         return s
 
     def addBuildRequest(self, brstatus):
         self.pendingBuilds.append(brstatus)
+        for w in self.watchers:
+            w.requestSubmitted(brstatus)
+
     def removeBuildRequest(self, brstatus):
         self.pendingBuilds.remove(brstatus)
 
     # buildStarted is called by our child BuildStatus instances
     def buildStarted(self, s):
         """Now the BuildStatus object is ready to go (it knows all of its
         Steps, its ETA, etc), so it is safe to notify our watchers."""
 
@@ -1670,35 +1747,42 @@ class BuilderStatus(styles.Versioned):
         assert s not in self.currentBuilds
         self.currentBuilds.append(s)
         self.addBuildToCache(s)
 
         # now that the BuildStatus is prepared to answer queries, we can
         # announce the new build to all our watchers
 
         for w in self.watchers: # TODO: maybe do this later? callLater(0)?
-            receiver = w.buildStarted(self.getName(), s)
-            if receiver:
-                if type(receiver) == type(()):
-                    s.subscribe(receiver[0], receiver[1])
-                else:
-                    s.subscribe(receiver)
-                d = s.waitUntilFinished()
-                d.addCallback(lambda s: s.unsubscribe(receiver))
-
+            try:
+                receiver = w.buildStarted(self.getName(), s)
+                if receiver:
+                    if type(receiver) == type(()):
+                        s.subscribe(receiver[0], receiver[1])
+                    else:
+                        s.subscribe(receiver)
+                    d = s.waitUntilFinished()
+                    d.addCallback(lambda s: s.unsubscribe(receiver))
+            except:
+                log.msg("Exception caught notifying %r of buildStarted event" % w)
+                log.err()
 
     def _buildFinished(self, s):
         assert s in self.currentBuilds
         s.saveYourself()
         self.currentBuilds.remove(s)
 
         name = self.getName()
         results = s.getResults()
         for w in self.watchers:
-            w.buildFinished(name, s, results)
+            try:
+                w.buildFinished(name, s, results)
+            except:
+                log.msg("Exception caught notifying %r of buildFinished event" % w)
+                log.err()
 
         self.prune() # conserve disk
 
 
     # waterfall display (history)
 
     # I want some kind of build event that holds everything about the build:
     # why, what changes went into it, the results of the build, itemized
@@ -1778,21 +1862,23 @@ class BuilderStatus(styles.Versioned):
             self.subscribers.remove(client)
 
 class SlaveStatus:
     implements(interfaces.ISlaveStatus)
 
     admin = None
     host = None
     connected = False
+    graceful_shutdown = False
 
     def __init__(self, name):
         self.name = name
         self._lastMessageReceived = 0
         self.runningBuilds = []
+        self.graceful_callbacks = []
 
     def getName(self):
         return self.name
     def getAdmin(self):
         return self.admin
     def getHost(self):
         return self.host
     def isConnected(self):
@@ -1811,16 +1897,35 @@ class SlaveStatus:
     def setLastMessageReceived(self, when):
         self._lastMessageReceived = when
 
     def buildStarted(self, build):
         self.runningBuilds.append(build)
     def buildFinished(self, build):
         self.runningBuilds.remove(build)
 
+    def getGraceful(self):
+        """Return the graceful shutdown flag"""
+        return self.graceful_shutdown
+    def setGraceful(self, graceful):
+        """Set the graceful shutdown flag, and notify all the watchers"""
+        self.graceful_shutdown = graceful
+        for cb in self.graceful_callbacks:
+            reactor.callLater(0, cb, graceful)
+    def addGracefulWatcher(self, watcher):
+        """Add watcher to the list of watchers to be notified when the
+        graceful shutdown flag is changed."""
+        if not watcher in self.graceful_callbacks:
+            self.graceful_callbacks.append(watcher)
+    def removeGracefulWatcher(self, watcher):
+        """Remove watcher from the list of watchers to be notified when the
+        graceful shutdown flag is changed."""
+        if watcher in self.graceful_callbacks:
+            self.graceful_callbacks.remove(watcher)
+
 class Status:
     """
     I represent the status of the buildmaster.
     """
     implements(interfaces.IStatus)
 
     def __init__(self, botmaster, basedir):
         """
@@ -1836,16 +1941,18 @@ class Status:
                         information (changes.pck, saved Build status
                         pickles) can be stored
         """
         self.botmaster = botmaster
         self.basedir = basedir
         self.watchers = []
         self.activeBuildSets = []
         assert os.path.isdir(basedir)
+        # compress logs bigger than 4k, a good default on linux
+        self.logCompressionLimit = 4*1024
 
 
     # methods called by our clients
 
     def getProjectName(self):
         return self.botmaster.parent.projectName
     def getProjectURL(self):
         return self.botmaster.parent.projectURL
@@ -2044,20 +2151,21 @@ class Status:
         # an unpickled object might not have category set from before,
         # so set it here to make sure
         builder_status.category = category
         builder_status.basedir = os.path.join(self.basedir, basedir)
         builder_status.name = name # it might have been updated
         builder_status.status = self
 
         if not os.path.isdir(builder_status.basedir):
-            os.mkdir(builder_status.basedir)
+            os.makedirs(builder_status.basedir)
         builder_status.determineNextBuildNumber()
 
         builder_status.setBigState("offline")
+        builder_status.setLogCompressionLimit(self.logCompressionLimit)
 
         for t in self.watchers:
             self.announceNewBuilder(t, name, builder_status)
 
         return builder_status
 
     def builderRemoved(self, name):
         for t in self.watchers:
--- a/buildbot/status/client.py
+++ b/buildbot/status/client.py
@@ -170,19 +170,16 @@ class RemoteBuild(pb.Referenceable):
         return self.b.getETA()
 
     def remote_getCurrentStep(self):
         return makeRemote(self.b.getCurrentStep())
 
     def remote_getText(self):
         return self.b.getText()
 
-    def remote_getColor(self):
-        return self.b.getColor()
-
     def remote_getResults(self):
         return self.b.getResults()
 
     def remote_getLogs(self):
         logs = {}
         for name,log in self.b.getLogs().items():
             logs[name] = IRemote(log)
         return logs
@@ -262,19 +259,16 @@ class RemoteBuildStep(pb.Referenceable):
         return self.s.waitUntilFinished() # returns a Deferred
 
     def remote_getETA(self):
         return self.s.getETA()
 
     def remote_getText(self):
         return self.s.getText()
 
-    def remote_getColor(self):
-        return self.s.getColor()
-
     def remote_getResults(self):
         return self.s.getResults()
 
 components.registerAdapter(RemoteBuildStep,
                            interfaces.IBuildStepStatus, IRemote)    
 
 class RemoteSlave:
     def __init__(self, slave):
@@ -295,18 +289,16 @@ components.registerAdapter(RemoteSlave,
 class RemoteEvent:
     def __init__(self, event):
         self.e = event
 
     def remote_getTimes(self):
         return self.s.getTimes()
     def remote_getText(self):
         return self.s.getText()
-    def remote_getColor(self):
-        return self.s.getColor()
 
 components.registerAdapter(RemoteEvent,
                            interfaces.IStatusEvent, IRemote)
 
 class RemoteLog(pb.Referenceable):
     def __init__(self, log):
         self.l = log
 
@@ -425,16 +417,20 @@ class StatusClientPerspective(base.Statu
     def perspective_getBuilder(self, name):
         b = self.status.getBuilder(name)
         return IRemote(b)
 
     def perspective_getSlave(self, name):
         s = self.status.getSlave(name)
         return IRemote(s)
 
+    def perspective_ping(self):
+        """Ping method to allow pb clients to validate their connections."""
+        return "pong"
+
     # IStatusReceiver methods, invoked if we've subscribed
 
     # mode >= builder
     def builderAdded(self, name, builder):
         self.client.callRemote("builderAdded", name, IRemote(builder))
         if self.subscribed in ("builds", "steps", "logs", "full"):
             self.subscribed_to_builders.append(name)
             return self
--- a/buildbot/status/mail.py
+++ b/buildbot/status/mail.py
@@ -1,11 +1,12 @@
 # -*- test-case-name: buildbot.test.test_status -*-
 
 # the email.MIMEMultipart module is only available in python-2.2.2 and later
+import re
 
 from email.Message import Message
 from email.Utils import formatdate
 from email.MIMEText import MIMEText
 try:
     from email.MIMEMultipart import MIMEMultipart
     canDoAttachments = True
 except ImportError:
@@ -14,28 +15,146 @@ import urllib
 
 from zope.interface import implements
 from twisted.internet import defer
 from twisted.mail.smtp import sendmail
 from twisted.python import log as twlog
 
 from buildbot import interfaces, util
 from buildbot.status import base
-from buildbot.status.builder import FAILURE, SUCCESS, WARNINGS
+from buildbot.status.builder import FAILURE, SUCCESS, WARNINGS, Results
+
+VALID_EMAIL = re.compile("[a-zA-Z0-9\.\_\%\-\+]+@[a-zA-Z0-9\.\_\%\-]+.[a-zA-Z]{2,6}")
+
+def message(attrs):
+    """Generate a buildbot mail message and return a tuple of message text
+    and type.
+
+    This function can be replaced using the customMesg variable in MailNotifier.
+    A message function will *always* get a dictionary of attributes with
+    the following values:
+
+      builderName - (str) Name of the builder that generated this event.
+      
+      projectName - (str) Name of the project.
+      
+      mode - (str) Mode set in MailNotifier. (failing, passing, problem).
+      
+      result - (str) Builder result as a string. 'success', 'warnings',
+               'failure', 'skipped', or 'exception'
+               
+      buildURL - (str) URL to build page.
+      
+      buildbotURL - (str) URL to buildbot main page.
+      
+      buildText - (str) Build text from build.getText().
+      
+      slavename - (str) Slavename.
+      
+      reason - (str) Build reason from build.getReason().
+      
+      responsibleUsers - (List of str) List of responsible users.
+      
+      branch - (str) Name of branch used. If no SourceStamp exists branch
+               is an empty string.
+               
+      revision - (str) Name of revision used. If no SourceStamp exists revision
+                 is an empty string.
+                 
+      patch - (str) Name of patch used. If no SourceStamp exists patch
+              is an empty string.
+
+      changes - (list of objs) List of change objects from SourceStamp. A change
+                object has the following useful information:
+                
+                who - who made this change
+                revision - what VC revision is this change
+                branch - on what branch did this change occur
+                when - when did this change occur
+                files - what files were affected in this change
+                comments - comments reguarding the change.
 
+                The functions asText and asHTML return a list of strings with
+                the above information formatted. 
+              
+      logs - (List of Tuples) List of tuples that contain the log name, log url
+             and log contents as a list of strings.
+    """
+    text = ""
+    if attrs['mode'] == "all":
+        text += "The Buildbot has finished a build"
+    elif attrs['mode'] == "failing":
+        text += "The Buildbot has detected a failed build"
+    elif attrs['mode'] == "passing":
+        text += "The Buildbot has detected a passing build"
+    else:
+        text += "The Buildbot has detected a new failure"
+    text += " of %s on %s.\n" % (attrs['builderName'], attrs['projectName'])
+    if attrs['buildURL']:
+        text += "Full details are available at:\n %s\n" % attrs['buildURL']
+    text += "\n"
+
+    if attrs['buildbotURL']:
+        text += "Buildbot URL: %s\n\n" % urllib.quote(attrs['buildbotURL'], '/:')
+
+    text += "Buildslave for this Build: %s\n\n" % attrs['slavename']
+    text += "Build Reason: %s\n" % attrs['reason']
+
+    #
+    # No source stamp
+    #
+    if attrs['branch']:
+        source = "unavailable"
+    else:
+        source = ""
+        if attrs['branch']:
+            source += "[branch %s] " % attrs['branch']
+        if attrs['revision']:
+            source += attrs['revision']
+        else:
+            source += "HEAD"
+        if attrs['patch']:
+            source += " (plus patch)"
+    text += "Build Source Stamp: %s\n" % source
+
+    text += "Blamelist: %s\n" % ",".join(attrs['responsibleUsers'])
+
+    text += "\n"
+
+    t = attrs['buildText']
+    if t:
+        t = ": " + " ".join(t)
+    else:
+        t = ""
+
+    if attrs['result'] == 'success':
+        text += "Build succeeded!\n"
+    elif attrs['result'] == 'warnings':
+        text += "Build Had Warnings%s\n" % t
+    else:
+        text += "BUILD FAILED%s\n" % t
+
+    text += "\n"
+    text += "sincerely,\n"
+    text += " -The Buildbot\n"
+    text += "\n"
+    return (text, 'plain')
 
 class Domain(util.ComparableMixin):
     implements(interfaces.IEmailLookup)
     compare_attrs = ["domain"]
 
     def __init__(self, domain):
         assert "@" not in domain
         self.domain = domain
 
     def getAddress(self, name):
+        """If name is already an email address, pass it through."""
+        if '@' in name:
+            return name
         return name + "@" + self.domain
 
 
 class MailNotifier(base.StatusReceiverMultiService):
     """This is a status notifier which sends email to a list of recipients
     upon the completion of each build. It can be configured to only send out
     mail for certain builds, and only send messages when the build fails, or
     when it transitions from success to failure. It can also be configured to
@@ -52,23 +171,23 @@ class MailNotifier(base.StatusReceiverMu
     different kinds of mail to different recipients, use multiple
     MailNotifiers.
     """
 
     implements(interfaces.IEmailSender)
 
     compare_attrs = ["extraRecipients", "lookup", "fromaddr", "mode",
                      "categories", "builders", "addLogs", "relayhost",
-                     "subject", "sendToInterestedUsers"]
+                     "subject", "sendToInterestedUsers", "customMesg"]
 
     def __init__(self, fromaddr, mode="all", categories=None, builders=None,
                  addLogs=False, relayhost="localhost",
                  subject="buildbot %(result)s in %(projectName)s on %(builder)s",
                  lookup=None, extraRecipients=[],
-                 sendToInterestedUsers=True):
+                 sendToInterestedUsers=True, customMesg=message):
         """
         @type  fromaddr: string
         @param fromaddr: the email address to be used in the 'From' header.
         @type  sendToInterestedUsers: boolean
         @param sendToInterestedUsers: if True (the default), send mail to all 
                                       of the Interested Users. If False, only
                                       send mail to the extraRecipients list.
 
@@ -122,38 +241,91 @@ class MailNotifier(base.StatusReceiverMu
                           provided, the notifier will only be able to send mail
                           to the addresses in the extraRecipients list. Most of
                           the time you can use a simple Domain instance. As a
                           shortcut, you can pass as string: this will be
                           treated as if you had provided Domain(str). For
                           example, lookup='twistedmatrix.com' will allow mail
                           to be sent to all developers whose SVN usernames
                           match their twistedmatrix.com account names.
+                          
+        @type  customMesg: func
+        @param customMesg: A function that returns a tuple containing the text of
+                           a custom message and its type. This function takes
+                           the dict attrs which has the following values:
+
+                           builderName - (str) Name of the builder that generated this event.
+      
+                           projectName - (str) Name of the project.
+                           
+                           mode - (str) Mode set in MailNotifier. (failing, passing, problem).
+      
+                           result - (str) Builder result as a string. 'success', 'warnings',
+                                    'failure', 'skipped', or 'exception'
+               
+                           buildURL - (str) URL to build page.
+      
+                           buildbotURL - (str) URL to buildbot main page.
+      
+                           buildText - (str) Build text from build.getText().
+      
+                           slavename - (str) Slavename.
+      
+                           reason - (str) Build reason from build.getReason().
+      
+                           responsibleUsers - (List of str) List of responsible users.
+      
+                           branch - (str) Name of branch used. If no SourceStamp exists branch
+                                    is an empty string.
+               
+                           revision - (str) Name of revision used. If no SourceStamp exists revision
+                                      is an empty string.
+                 
+                           patch - (str) Name of patch used. If no SourceStamp exists patch
+                                   is an empty string.
+
+                           changes - (list of objs) List of change objects from SourceStamp. A change
+                                     object has the following useful information:
+                
+                                     who - who made this change
+                                     revision - what VC revision is this change
+                                     branch - on what branch did this change occur
+                                     when - when did this change occur
+                                     files - what files were affected in this change
+                                     comments - comments reguarding the change.
+
+                                     The functions asText and asHTML return a list of strings with
+                                     the above information formatted. 
+
+                           logs - (List of Tuples) List of tuples that contain the log name, log url,
+                                  and log contents as a list of strings.
+
         """
 
         base.StatusReceiverMultiService.__init__(self)
         assert isinstance(extraRecipients, (list, tuple))
         for r in extraRecipients:
             assert isinstance(r, str)
-            assert "@" in r # require full email addresses, not User names
+            assert VALID_EMAIL.search(r) # require full email addresses, not User names
         self.extraRecipients = extraRecipients
         self.sendToInterestedUsers = sendToInterestedUsers
         self.fromaddr = fromaddr
         assert mode in ('all', 'failing', 'problem')
         self.mode = mode
         self.categories = categories
         self.builders = builders
         self.addLogs = addLogs
         self.relayhost = relayhost
         self.subject = subject
         if lookup is not None:
             if type(lookup) is str:
                 lookup = Domain(lookup)
             assert interfaces.IEmailLookup.providedBy(lookup)
         self.lookup = lookup
+        self.customMesg = customMesg
         self.watched = []
         self.status = None
 
         # you should either limit on builders or categories, not both
         if self.builders != None and self.categories != None:
             twlog.err("Please specify only builders to ignore or categories to include")
             raise # FIXME: the asserts above do not raise some Exception either
 
@@ -212,150 +384,141 @@ class MailNotifier(base.StatusReceiverMu
         # when the mail has been sent. To help unit tests, we return that
         # Deferred here even though the normal IStatusReceiver.buildFinished
         # signature doesn't do anything with it. If that changes (if
         # .buildFinished's return value becomes significant), we need to
         # rearrange this.
         return self.buildMessage(name, build, results)
 
     def buildMessage(self, name, build, results):
-        projectName = self.status.getProjectName()
-        text = ""
-        if self.mode == "all":
-            text += "The Buildbot has finished a build"
-        elif self.mode == "failing":
-            text += "The Buildbot has detected a failed build"
-        elif self.mode == "passing":
-            text += "The Buildbot has detected a passing build"
-        else:
-            text += "The Buildbot has detected a new failure"
-        text += " of %s on %s.\n" % (name, projectName)
-        buildurl = self.status.getURLForThing(build)
-        if buildurl:
-            text += "Full details are available at:\n %s\n" % buildurl
-        text += "\n"
+        #
+        # logs is a list of tuples that contain the log
+        # name, log url, and the log contents as a list of strings.
+        #
+        logs = list()
+        for log in build.getLogs():
+            stepName = log.getStep().getName()
+            logName = log.getName()
+            logs.append(('%s.%s' % (stepName, logName),
+                         '%s/steps/%s/logs/%s' % (self.status.getURLForThing(build), stepName, logName),
+                         log.getText().splitlines()))
+                
+        attrs = {'builderName': name,
+                 'projectName': self.status.getProjectName(),
+                 'mode': self.mode,
+                 'result': Results[results],
+                 'buildURL': self.status.getURLForThing(build),
+                 'buildbotURL': self.status.getBuildbotURL(),
+                 'buildText': build.getText(),
+                 'slavename': build.getSlavename(),
+                 'reason':  build.getReason(),
+                 'responsibleUsers': build.getResponsibleUsers(),
+                 'branch': "",
+                 'revision': "",
+                 'patch': "",
+                 'changes': [],
+                 'logs': logs}
 
-        url = self.status.getBuildbotURL()
-        if url:
-            text += "Buildbot URL: %s\n\n" % urllib.quote(url, '/:')
-
-        text += "Buildslave for this Build: %s\n\n" % build.getSlavename()
-        text += "Build Reason: %s\n" % build.getReason()
-
-        patch = None
         ss = build.getSourceStamp()
-        if ss is None:
-            source = "unavailable"
-        else:
-            source = ""
-            if ss.branch:
-                source += "[branch %s] " % ss.branch
-            if ss.revision:
-                source += ss.revision
-            else:
-                source += "HEAD"
-            if ss.patch is not None:
-                source += " (plus patch)"
-                patch = ss.patch
-        text += "Build Source Stamp: %s\n" % source
-
-        text += "Blamelist: %s\n" % ",".join(build.getResponsibleUsers())
-
-        # TODO: maybe display changes here? or in an attachment?
-        text += "\n"
+        if ss:
+            attrs['branch'] = ss.branch
+            attrs['revision'] = ss.revision
+            attrs['patch'] = ss.patch
+            attrs['changes'] = ss.changes[:]
 
-        t = build.getText()
-        if t:
-            t = ": " + " ".join(t)
-        else:
-            t = ""
-
-        if results == SUCCESS:
-            text += "Build succeeded!\n"
-            res = "success"
-        elif results == WARNINGS:
-            text += "Build Had Warnings%s\n" % t
-            res = "warnings"
-        else:
-            text += "BUILD FAILED%s\n" % t
-            res = "failure"
-
-        if self.addLogs and build.getLogs():
-            text += "Logs are attached.\n"
-
-        # TODO: it would be nice to provide a URL for the specific build
-        # here. That involves some coordination with html.Waterfall .
-        # Ideally we could do:
-        #  helper = self.parent.getServiceNamed("html")
-        #  if helper:
-        #      url = helper.getURLForBuild(build)
-
-        text += "\n"
-        text += "sincerely,\n"
-        text += " -The Buildbot\n"
-        text += "\n"
+        text, type = self.customMesg(attrs)
+        assert type in ('plain', 'html'), "'%s' message type must be 'plain' or 'html'." % type
 
         haveAttachments = False
-        if patch or self.addLogs:
+        if attrs['patch'] or self.addLogs:
             haveAttachments = True
             if not canDoAttachments:
                 twlog.msg("warning: I want to send mail with attachments, "
                           "but this python is too old to have "
                           "email.MIMEMultipart . Please upgrade to python-2.3 "
                           "or newer to enable addLogs=True")
 
         if haveAttachments and canDoAttachments:
             m = MIMEMultipart()
-            m.attach(MIMEText(text))
+            m.attach(MIMEText(text, type))
         else:
             m = Message()
             m.set_payload(text)
+            m.set_type("text/%s" % type)
 
         m['Date'] = formatdate(localtime=True)
-        m['Subject'] = self.subject % { 'result': res,
-                                        'projectName': projectName,
-                                        'builder': name,
+        m['Subject'] = self.subject % { 'result': attrs['result'],
+                                        'projectName': attrs['projectName'],
+                                        'builder': attrs['builderName'],
                                         }
         m['From'] = self.fromaddr
         # m['To'] is added later
 
-        if patch:
-            a = MIMEText(patch)
+        if attrs['patch']:
+            a = MIMEText(attrs['patch'][1])
             a.add_header('Content-Disposition', "attachment",
                          filename="source patch")
             m.attach(a)
         if self.addLogs:
             for log in build.getLogs():
                 name = "%s.%s" % (log.getStep().getName(),
                                   log.getName())
-                a = MIMEText(log.getText())
-                a.add_header('Content-Disposition', "attachment",
-                             filename=name)
-                m.attach(a)
+                if self._shouldAttachLog(log.getName()) or self._shouldAttachLog(name):
+                    a = MIMEText(log.getText())
+                    a.add_header('Content-Disposition', "attachment",
+                                 filename=name)
+                    m.attach(a)
 
         # now, who is this message going to?
         dl = []
-        recipients = self.extraRecipients[:]
+        recipients = []
         if self.sendToInterestedUsers and self.lookup:
             for u in build.getInterestedUsers():
                 d = defer.maybeDeferred(self.lookup.getAddress, u)
                 d.addCallback(recipients.append)
                 dl.append(d)
         d = defer.DeferredList(dl)
         d.addCallback(self._gotRecipients, recipients, m)
         return d
 
+    def _shouldAttachLog(self, logname):
+        if type(self.addLogs) is bool:
+            return self.addLogs
+        return logname in self.addLogs
+
     def _gotRecipients(self, res, rlist, m):
-        recipients = []
+        recipients = set()
+
         for r in rlist:
-            if r is not None and r not in recipients:
-                recipients.append(r)
-        recipients.sort()
-        m['To'] = ", ".join(recipients)
-        return self.sendMessage(m, recipients)
+            if r is None: # getAddress didn't like this address
+                continue
+
+            # Git can give emails like 'User' <user@foo.com>@foo.com so check
+            # for two @ and chop the last
+            if r.count('@') > 1:
+                r = r[:r.rindex('@')]
+
+            if VALID_EMAIL.search(r):
+                recipients.add(r)
+            else:
+                twlog.msg("INVALID EMAIL: %r" + r)
+
+        # if we're sending to interested users move the extra's to the CC
+        # list so they can tell if they are also interested in the change
+        # unless there are no interested users
+        if self.sendToInterestedUsers and len(recipients):
+            m['CC'] = ", ".join(sorted(self.extraRecipients[:]))
+        else:
+            [recipients.add(r) for r in self.extraRecipients[:]]
+
+        m['To'] = ", ".join(sorted(recipients))
+
+        # The extras weren't part of the TO list so add them now
+        if self.sendToInterestedUsers:
+            for r in self.extraRecipients:
+                recipients.add(r)
+
+        return self.sendMessage(m, list(recipients))
 
     def sendMessage(self, m, recipients):
         s = m.as_string()
-        ds = []
         twlog.msg("sending mail (%d bytes) to" % len(s), recipients)
-        for recip in recipients:
-            ds.append(sendmail(self.relayhost, self.fromaddr, recip, s))
-        return defer.DeferredList(ds)
+        return sendmail(self.relayhost, self.fromaddr, recipients, s)
--- a/buildbot/status/web/base.py
+++ b/buildbot/status/web/base.py
@@ -1,15 +1,15 @@
 
 import urlparse, urllib, time
 from zope.interface import Interface
 from twisted.web import html, resource
 from buildbot.status import builder
-from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
-
+from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION
+from buildbot import version, util
 
 class ITopBox(Interface):
     """I represent a box in the top row of the waterfall display: the one
     which shows the status of the last build for each builder."""
     def getBox(self, request):
         """Return a Box instance, which can produce a <td> cell.
         """
 
@@ -27,16 +27,17 @@ class IBox(Interface):
         """
 
 class IHTMLLog(Interface):
     pass
 
 css_classes = {SUCCESS: "success",
                WARNINGS: "warnings",
                FAILURE: "failure",
+               SKIPPED: "skipped",
                EXCEPTION: "exception",
                }
 
 ROW_TEMPLATE = '''
 <div class="row">
   <span class="label">%(label)s</span>
   <span class="field">%(field)s</span>
 </div>
@@ -84,35 +85,30 @@ def make_force_build_form(forceURL, on_a
       + make_row("Reason for build:",
                  "<input type='text' name='comments' />")
       + make_row("Branch to build:",
                  "<input type='text' name='branch' />")
       + make_row("Revision to build:",
                  "<input type='text' name='revision' />")
       + '<input type="submit" value="Force Build" /></form>\n')
 
-colormap = {
-    'green': '#72ff75',
-    }
 def td(text="", parms={}, **props):
     data = ""
     data += "  "
     #if not props.has_key("border"):
     #    props["border"] = 1
     props.update(parms)
-    if props.has_key("bgcolor"):
-        props["bgcolor"] = colormap.get(props["bgcolor"], props["bgcolor"])
     comment = props.get("comment", None)
     if comment:
         data += "<!-- %s -->" % comment
     data += "<td"
     class_ = props.get('class_', None)
     if class_:
         props["class"] = class_
-    for prop in ("align", "bgcolor", "colspan", "rowspan", "border",
+    for prop in ("align", "colspan", "rowspan", "border",
                  "valign", "halign", "class"):
         p = props.get(prop, None)
         if p != None:
             data += " %s=\"%s\"" % (prop, p)
     data += ">"
     if not text:
         text = "&nbsp;"
     if isinstance(text, list):
@@ -165,42 +161,46 @@ def path_to_builder(request, builderstat
 def path_to_build(request, buildstatus):
     return (path_to_builder(request, buildstatus.getBuilder()) +
             "/builds/%d" % buildstatus.getNumber())
 
 def path_to_step(request, stepstatus):
     return (path_to_build(request, stepstatus.getBuild()) +
             "/steps/%s" % urllib.quote(stepstatus.getName(), safe=''))
 
+def path_to_slave(request, slave):
+    return (path_to_root(request) +
+            "buildslaves/" +
+            urllib.quote(slave.getName(), safe=''))
+
 class Box:
     # a Box wraps an Event. The Box has HTML <td> parameters that Events
     # lack, and it has a base URL to which each File's name is relative.
     # Events don't know about HTML.
     spacer = False
-    def __init__(self, text=[], color=None, class_=None, urlbase=None,
+    def __init__(self, text=[], class_=None, urlbase=None,
                  **parms):
         self.text = text
-        self.color = color
         self.class_ = class_
         self.urlbase = urlbase
         self.show_idle = 0
         if parms.has_key('show_idle'):
             del parms['show_idle']
             self.show_idle = 1
 
         self.parms = parms
         # parms is a dict of HTML parameters for the <td> element that will
         # represent this Event in the waterfall display.
 
     def td(self, **props):
         props.update(self.parms)
         text = self.text
         if not text and self.show_idle:
             text = ["[idle]"]
-        return td(text, props, bgcolor=self.color, class_=self.class_)
+        return td(text, props, class_=self.class_)
 
 
 class HtmlResource(resource.Resource):
     # this is a cheap sort of template thingy
     contentType = "text/html; charset=UTF-8"
     title = "Buildbot"
     addSlash = False # adapted from Nevow
 
@@ -253,21 +253,49 @@ class HtmlResource(resource.Resource):
         return data
 
     def getStatus(self, request):
         return request.site.buildbot_service.getStatus()
     def getControl(self, request):
         return request.site.buildbot_service.getControl()
 
     def getChangemaster(self, request):
-        return request.site.buildbot_service.parent.change_svc
+        return request.site.buildbot_service.getChangeSvc()
 
     def path_to_root(self, request):
         return path_to_root(request)
 
+    def footer(self, s, req):
+        # TODO: this stuff should be generated by a template of some sort
+        projectURL = s.getProjectURL()
+        projectName = s.getProjectName()
+        data = '<hr /><div class="footer">\n'
+
+        welcomeurl = self.path_to_root(req) + "index.html"
+        data += '[<a href="%s">welcome</a>]\n' % welcomeurl
+        data += "<br />\n"
+
+        data += '<a href="http://buildbot.sourceforge.net/">Buildbot</a>'
+        data += "-%s " % version
+        if projectName:
+            data += "working for the "
+            if projectURL:
+                data += "<a href=\"%s\">%s</a> project." % (projectURL,
+                                                            projectName)
+            else:
+                data += "%s project." % projectName
+        data += "<br />\n"
+        data += ("Page built: " +
+                 time.strftime("%a %d %b %Y %H:%M:%S",
+                               time.localtime(util.now()))
+                 + "\n")
+        data += '</div>\n'
+
+        return data
+
     def getTitle(self, request):
         return self.title
 
     def fillTemplate(self, template, request):
         s = request.site.buildbot_service
         values = s.template_values.copy()
         values['root'] = self.path_to_root(request)
         # e.g. to reference the top-level 'buildbot.css' page, use
@@ -344,29 +372,27 @@ class OneLineMixin:
         text = build.getText()
         try:
             rev = build.getProperty("got_revision")
             if rev is None:
                 rev = "??"
         except KeyError:
             rev = "??"
         rev = str(rev)
-        if len(rev) > 20:
+        if len(rev) > 40:
             rev = "version is too-long"
         root = self.path_to_root(req)
         css_class = css_classes.get(results, "")
         values = {'class': css_class,
                   'builder_name': builder_name,
                   'buildnum': build.getNumber(),
                   'results': css_class,
                   'text': " ".join(build.getText()),
-                  'buildurl': (root +
-                               "builders/%s/builds/%d" % (builder_name,
-                                                          build.getNumber())),
-                  'builderurl': (root + "builders/%s" % builder_name),
+                  'buildurl': path_to_build(req, build),
+                  'builderurl': path_to_builder(req, build.getBuilder()),
                   'rev': rev,
                   'time': time.strftime(self.LINE_TIME_FORMAT,
                                         time.localtime(build.getTimes()[0])),
                   }
         return values
 
     def make_line(self, req, build, include_builder=True):
         '''
--- a/buildbot/status/web/baseweb.py
+++ b/buildbot/status/web/baseweb.py
@@ -8,16 +8,18 @@ from twisted.application import strports
 from twisted.web import server, distrib, static, html
 from twisted.spread import pb
 
 from buildbot.interfaces import IControl, IStatusReceiver
 
 from buildbot.status.web.base import HtmlResource, Box, \
      build_get_class, ICurrentBox, OneLineMixin, map_branches, \
      make_stop_form, make_force_build_form
+from buildbot.status.web.feeds import Rss20StatusResource, \
+     Atom10StatusResource
 from buildbot.status.web.waterfall import WaterfallStatusResource
 from buildbot.status.web.grid import GridStatusResource
 from buildbot.status.web.changes import ChangesResource
 from buildbot.status.web.builder import BuildersResource
 from buildbot.status.web.slaves import BuildSlavesResource
 from buildbot.status.web.xmlrpc import XMLRPCServer
 from buildbot.status.web.about import AboutBuildbot
 
@@ -212,17 +214,17 @@ class OneBoxPerBuilder(HtmlResource):
                 try:
                     label = b.getProperty("got_revision")
                 except KeyError:
                     label = None
                 if not label or len(str(label)) > 20:
                     label = "#%d" % b.getNumber()
                 text = ['<a href="%s">%s</a>' % (url, label)]
                 text.extend(b.getText())
-                box = Box(text, b.getColor(),
+                box = Box(text,
                           class_="LastBuild box %s" % build_get_class(b))
                 data += box.td(align="center")
             else:
                 data += '<td class="LastBuild box" >no build</td>\n'
             current_box = ICurrentBox(builder).getBox(status)
             data += current_box.td(align="center")
 
             builder_status = builder.getState()[0]
@@ -280,16 +282,22 @@ class WebStatus(service.MultiService):
 
     """
     The webserver provided by this class has the following resources:
 
      /waterfall : the big time-oriented 'waterfall' display, with links
                   to individual changes, builders, builds, steps, and logs.
                   A number of query-arguments can be added to influence
                   the display.
+     /rss : a rss feed summarizing all failed builds. The same
+            query-arguments used by 'waterfall' can be added to
+            influence the feed output.
+     /atom : an atom feed summarizing all failed builds. The same
+             query-arguments used by 'waterfall' can be added to
+             influence the feed output.
      /grid : another summary display that shows a grid of builds, with
              sourcestamps on the x axis, and builders on the y.  Query 
              arguments similar to those for the waterfall can be added.
      /builders/BUILDERNAME: a page summarizing the builder. This includes
                             references to the Schedulers that feed it,
                             any builds currently in the queue, which
                             buildslaves are designated or attached, and a
                             summary of the build process it uses.
@@ -355,17 +363,17 @@ class WebStatus(service.MultiService):
 
     # we are not a ComparableMixin, and therefore the webserver will be
     # rebuilt every time we reconfig. This is because WebStatus.putChild()
     # makes it too difficult to tell whether two instances are the same or
     # not (we'd have to do a recursive traversal of all children to discover
     # all the changes).
 
     def __init__(self, http_port=None, distrib_port=None, allowForce=False,
-                       public_html="public_html"):
+                       public_html="public_html", site=None):
         """Run a web server that provides Buildbot status.
 
         @type  http_port: int or L{twisted.application.strports} string
         @param http_port: a strports specification describing which port the
                           buildbot should use for its web server, with the
                           Waterfall display as the root page. For backwards
                           compatibility this can also be an int. Use
                           'tcp:8000' to listen on that port, or
@@ -395,35 +403,43 @@ class WebStatus(service.MultiService):
                              the strports parser.
 
         @param allowForce: boolean, if True then the webserver will allow
                            visitors to trigger and cancel builds
 
         @param public_html: the path to the public_html directory for this display,
                             either absolute or relative to the basedir.  The default
                             is 'public_html', which selects BASEDIR/public_html.
+
+        @type site: None or L{twisted.web.server.Site}
+        @param site: Use this if you want to define your own object instead of
+                     using the default.`
         """
 
         service.MultiService.__init__(self)
         if type(http_port) is int:
             http_port = "tcp:%d" % http_port
         self.http_port = http_port
         if distrib_port is not None:
             if type(distrib_port) is int:
                 distrib_port = "tcp:%d" % distrib_port
             if distrib_port[0] in "/~.": # pathnames
                 distrib_port = "unix:%s" % distrib_port
         self.distrib_port = distrib_port
         self.allowForce = allowForce
         self.public_html = public_html
 
-        # this will be replaced once we've been attached to a parent (and
-        # thus have a basedir and can reference BASEDIR)
-        root = static.Data("placeholder", "text/plain")
-        self.site = server.Site(root)
+        # If we were given a site object, go ahead and use it.
+        if site:
+            self.site = site
+        else:
+            # this will be replaced once we've been attached to a parent (and
+            # thus have a basedir and can reference BASEDIR)
+            root = static.Data("placeholder", "text/plain")
+            self.site = server.Site(root)
         self.childrenToBeAdded = {}
 
         self.setupUsualPages()
 
         # the following items are accessed by HtmlResource when it renders
         # each page.
         self.site.buildbot_service = self
         self.header = HEADER
@@ -464,36 +480,47 @@ class WebStatus(service.MultiService):
         if self.distrib_port is None:
             return "<WebStatus on port %s at %s>" % (self.http_port,
                                                      hex(id(self)))
         return ("<WebStatus on port %s and path %s at %s>" %
                 (self.http_port, self.distrib_port, hex(id(self))))
 
     def setServiceParent(self, parent):
         service.MultiService.setServiceParent(self, parent)
+
+        # this class keeps a *separate* link to the buildmaster, rather than
+        # just using self.parent, so that when we are "disowned" (and thus
+        # parent=None), any remaining HTTP clients of this WebStatus will still
+        # be able to get reasonable results.
+        self.master = parent
+
         self.setupSite()
 
     def setupSite(self):
         # this is responsible for creating the root resource. It isn't done
         # at __init__ time because we need to reference the parent's basedir.
-        htmldir = os.path.abspath(os.path.join(self.parent.basedir, self.public_html))
+        htmldir = os.path.abspath(os.path.join(self.master.basedir, self.public_html))
         if os.path.isdir(htmldir):
             log.msg("WebStatus using (%s)" % htmldir)
         else:
             log.msg("WebStatus: warning: %s is missing. Do you need to run"
                     " 'buildbot upgrade-master' on this buildmaster?" % htmldir)
             # all static pages will get a 404 until upgrade-master is used to
             # populate this directory. Create the directory, though, since
             # otherwise we get internal server errors instead of 404s.
             os.mkdir(htmldir)
         root = static.File(htmldir)
 
         for name, child_resource in self.childrenToBeAdded.iteritems():
             root.putChild(name, child_resource)
 
+        status = self.getStatus()
+        root.putChild("rss", Rss20StatusResource(status))
+        root.putChild("atom", Atom10StatusResource(status))
+
         self.site.resource = root
 
     def putChild(self, name, child_resource):
         """This behaves a lot like root.putChild() . """
         self.childrenToBeAdded[name] = child_resource
 
     def registerChannel(self, channel):
         self.channels[channel] = 1 # weakrefs
@@ -504,22 +531,25 @@ class WebStatus(service.MultiService):
                 channel.transport.loseConnection()
             except:
                 log.msg("WebStatus.stopService: error while disconnecting"
                         " leftover clients")
                 log.err()
         return service.MultiService.stopService(self)
 
     def getStatus(self):
-        return self.parent.getStatus()
+        return self.master.getStatus()
+
     def getControl(self):
         if self.allowForce:
-            return IControl(self.parent)
+            return IControl(self.master)
         return None
 
+    def getChangeSvc(self):
+        return self.master.change_svc
     def getPortnum(self):
         # this is for the benefit of unit tests
         s = list(self)[0]
         return s._port.getHost().port
 
 # resources can get access to the IStatus by calling
 # request.site.buildbot_service.getStatus()
 
--- a/buildbot/status/web/build.py
+++ b/buildbot/status/web/build.py
@@ -1,17 +1,17 @@
 
 from twisted.web import html
 from twisted.web.util import Redirect, DeferredResource
 from twisted.internet import defer, reactor
 
 import urllib, time
 from twisted.python import log
 from buildbot.status.web.base import HtmlResource, make_row, make_stop_form, \
-     css_classes, path_to_builder
+     css_classes, path_to_builder, path_to_slave
 
 from buildbot.status.web.tests import TestsResource
 from buildbot.status.web.step import StepsResource
 from buildbot import version, util
 
 # /builders/$builder/builds/$buildnum
 class StatusResourceBuild(HtmlResource):
     addSlash = True
@@ -29,17 +29,16 @@ class StatusResourceBuild(HtmlResource):
 
     def body(self, req):
         b = self.build_status
         status = self.getStatus(req)
         projectURL = status.getProjectURL()
         projectName = status.getProjectName()
         data = ('<div class="title"><a href="%s">%s</a></div>\n'
                 % (self.path_to_root(req), projectName))
-        # the color in the following line gives python-mode trouble
         builder_name = b.getBuilder().getName()
         data += ("<h1><a href=\"%s\">Builder %s</a>: Build #%d</h1>\n"
                  % (path_to_builder(req, b.getBuilder()),
                     builder_name, b.getNumber()))
 
         if not b.isFinished():
             data += "<h2>Build In Progress</h2>"
             when = b.getETA()
@@ -85,17 +84,21 @@ class StatusResourceBuild(HtmlResource):
             got_revision = str(got_revision)
             if len(got_revision) > 40:
                 got_revision = "[revision string too long]"
             data += "  <li>Got Revision: %s</li>\n" % got_revision
         data += " </ul>\n"
 
         # TODO: turn this into a table, or some other sort of definition-list
         # that doesn't take up quite so much vertical space
-        data += "<h2>Buildslave:</h2>\n %s\n" % html.escape(b.getSlavename())
+        try:
+            slaveurl = path_to_slave(req, status.getSlave(b.getSlavename()))
+            data += "<h2>Buildslave:</h2>\n <a href=\"%s\">%s</a>\n" % (html.escape(slaveurl), html.escape(b.getSlavename()))
+        except KeyError:
+            data += "<h2>Buildslave:</h2>\n %s\n" % html.escape(b.getSlavename())
         data += "<h2>Reason:</h2>\n%s\n" % html.escape(b.getReason())
 
         data += "<h2>Steps and Logfiles:</h2>\n"
         # TODO:
 #        urls = self.original.getURLs()
 #        ex_url_class = "BuildStep external"
 #        for name, target in urls.items():
 #            text.append('[<a href="%s" class="%s">%s</a>]' %
--- a/buildbot/status/web/builder.py
+++ b/buildbot/status/web/builder.py
@@ -2,20 +2,19 @@
 from twisted.web.error import NoResource
 from twisted.web import html, static
 from twisted.web.util import Redirect
 
 import re, urllib, time
 from twisted.python import log
 from buildbot import interfaces
 from buildbot.status.web.base import HtmlResource, make_row, \
-     make_force_build_form, OneLineMixin
+     make_force_build_form, OneLineMixin, path_to_build, path_to_slave, path_to_builder
 from buildbot.process.base import BuildRequest
 from buildbot.sourcestamp import SourceStamp
-from buildbot import version, util
 
 from buildbot.status.web.build import BuildsResource, StatusResourceBuild
 
 # /builders/$builder
 class StatusResourceBuilder(HtmlResource, OneLineMixin):
     addSlash = True
 
     def __init__(self, builder_status, builder_control):
@@ -23,33 +22,33 @@ class StatusResourceBuilder(HtmlResource
         self.builder_status = builder_status
         self.builder_control = builder_control
 
     def getTitle(self, request):
         return "Buildbot: %s" % html.escape(self.builder_status.getName())
 
     def build_line(self, build, req):
         buildnum = build.getNumber()
-        buildurl = req.childLink("builds/%d" % buildnum)
+        buildurl = path_to_build(req, build)
         data = '<a href="%s">#%d</a> ' % (buildurl, buildnum)
 
         when = build.getETA()
         if when is not None:
             when_time = time.strftime("%H:%M:%S",
                                       time.localtime(time.time() + when))
             data += "ETA %ds (%s) " % (when, when_time)
         step = build.getCurrentStep()
         if step:
             data += "[%s]" % step.getName()
         else:
             data += "[waiting for Lock]"
             # TODO: is this necessarily the case?
 
         if self.builder_control is not None:
-            stopURL = urllib.quote(req.childLink("builds/%d/stop" % buildnum))
+            stopURL = path_to_build(req, build) + '/stop'
             data += '''
 <form action="%s" class="command stopbuild" style="display:inline">
   <input type="submit" value="Stop Build" />
 </form>''' % stopURL
         return data
 
     def body(self, req):
         b = self.builder_status
@@ -88,72 +87,50 @@ class StatusResourceBuilder(HtmlResource
                 data += "<br />\n" # separator
                 # TODO: or empty list?
         data += "</ul>\n"
 
 
         data += "<h2>Buildslaves:</h2>\n"
         data += "<ol>\n"
         for slave in slaves:
-            data += "<li><b>%s</b>: " % html.escape(slave.getName())
+            slaveurl = path_to_slave(req, slave)
+            data += "<li><b><a href=\"%s\">%s</a></b>: " % (html.escape(slaveurl), html.escape(slave.getName()))
             if slave.isConnected():
                 data += "CONNECTED\n"
                 if slave.getAdmin():
                     data += make_row("Admin:", html.escape(slave.getAdmin()))
                 if slave.getHost():
                     data += "<span class='label'>Host info:</span>\n"
                     data += html.PRE(slave.getHost())
             else:
                 data += ("NOT CONNECTED\n")
             data += "</li>\n"
         data += "</ol>\n"
 
         if control is not None and connected_slaves:
-            forceURL = urllib.quote(req.childLink("force"))
+            forceURL = path_to_builder(req, b) + '/force'
             data += make_force_build_form(forceURL)
         elif control is not None:
             data += """
             <p>All buildslaves appear to be offline, so it's not possible
             to force this build to execute at this time.</p>
             """
 
         if control is not None:
-            pingURL = urllib.quote(req.childLink("ping"))
+            pingURL = path_to_builder(req, b) + '/ping'
             data += """
             <form action="%s" class='command pingbuilder'>
             <p>To ping the buildslave(s), push the 'Ping' button</p>
 
             <input type="submit" value="Ping Builder" />
             </form>
             """ % pingURL
 
-        # TODO: this stuff should be generated by a template of some sort
-        projectURL = status.getProjectURL()
-        projectName = status.getProjectName()
-        data += '<hr /><div class="footer">\n'
-
-        welcomeurl = self.path_to_root(req) + "index.html"
-        data += '[<a href="%s">welcome</a>]\n' % welcomeurl
-        data += "<br />\n"
-
-        data += '<a href="http://buildbot.sourceforge.net/">Buildbot</a>'
-        data += "-%s " % version
-        if projectName:
-            data += "working for the "
-            if projectURL:
-                data += "<a href=\"%s\">%s</a> project." % (projectURL,
-                                                            projectName)
-            else:
-                data += "%s project." % projectName
-        data += "<br />\n"
-        data += ("Page built: " +
-                 time.strftime("%a %d %b %Y %H:%M:%S",
-                               time.localtime(util.now()))
-                 + "\n")
-        data += '</div>\n'
+        data += self.footer(status, req)
 
         return data
 
     def force(self, req):
         """
 
         Custom properties can be passed from the web form.  To do
         this, subclass this class, overriding the force() method.  You
@@ -306,43 +283,20 @@ class BuildersResource(HtmlResource):
 
         # TODO: this is really basic. It should be expanded to include a
         # brief one-line summary of the builder (perhaps with whatever the
         # builder is currently doing)
         data += "<ol>\n"
         for bname in s.getBuilderNames():
             data += (' <li><a href="%s">%s</a></li>\n' %
                      (req.childLink(urllib.quote(bname, safe='')),
-                      urllib.quote(bname, safe='')))
+                      bname))
         data += "</ol>\n"
 
-        # TODO: this stuff should be generated by a template of some sort
-        projectURL = s.getProjectURL()
-        projectName = s.getProjectName()
-        data += '<hr /><div class="footer">\n'
-
-        welcomeurl = self.path_to_root(req) + "index.html"
-        data += '[<a href="%s">welcome</a>]\n' % welcomeurl
-        data += "<br />\n"
-
-        data += '<a href="http://buildbot.sourceforge.net/">Buildbot</a>'
-        data += "-%s " % version
-        if projectName:
-            data += "working for the "
-            if projectURL:
-                data += "<a href=\"%s\">%s</a> project." % (projectURL,
-                                                            projectName)
-            else:
-                data += "%s project." % projectName
-        data += "<br />\n"
-        data += ("Page built: " +
-                 time.strftime("%a %d %b %Y %H:%M:%S",
-                               time.localtime(util.now()))
-                 + "\n")
-        data += '</div>\n'
+        data += self.footer(s, req)
 
         return data
 
     def getChild(self, path, req):
         s = self.getStatus(req)
         if path in s.getBuilderNames():
             builder_status = s.getBuilder(path)
             builder_control = None
--- a/buildbot/status/web/changes.py
+++ b/buildbot/status/web/changes.py
@@ -31,11 +31,11 @@ class ChangesResource(HtmlResource):
 
 
 class ChangeBox(components.Adapter):
     implements(IBox)
 
     def getBox(self, req):
         url = req.childLink("../changes/%d" % self.original.number)
         text = self.original.get_HTML_box(url)
-        return Box([text], color="white", class_="Change")
+        return Box([text], class_="Change")
 components.registerAdapter(ChangeBox, Change, IBox)
 
--- a/buildbot/status/web/classic.css
+++ b/buildbot/status/web/classic.css
@@ -8,17 +8,17 @@ td.Event, td.BuildStep, td.Activity, td.
 }
 
 td.box {
        border: 1px solid;
 }
 
 /* Activity states */
 .offline { 
-        background-color: red;
+        background-color: gray;
 }
 .idle {
 	background-color: white;
 }
 .waiting { 
         background-color: yellow;
 }
 .building { 
--- a/buildbot/status/web/grid.py
+++ b/buildbot/status/web/grid.py
@@ -1,16 +1,19 @@
 from __future__ import generators
 
 import sys, time, os.path
 import urllib
 
 from buildbot import util
 from buildbot import version
 from buildbot.status.web.base import HtmlResource
+#from buildbot.status.web.base import Box, HtmlResource, IBox, ICurrentBox, \
+#     ITopBox, td, build_get_class, path_to_build, path_to_step, map_branches
+from buildbot.status.web.base import build_get_class
 
 # set grid_css to the full pathname of the css file
 if hasattr(sys, "frozen"):
     # all 'data' files are in the directory of our executable
     here = os.path.dirname(sys.executable)
     grid_css = os.path.abspath(os.path.join(here, "grid.css"))
 else:
     # running from source; look for a sibling to __file__
@@ -36,17 +39,17 @@ class GridStatusResource(HtmlResource):
         p = status.getProjectName()
         if p:
             return "BuildBot: %s" % p
         else:
             return "BuildBot"
 
     def getChangemaster(self, request):
         # TODO: this wants to go away, access it through IStatus
-        return request.site.buildbot_service.parent.change_svc
+        return request.site.buildbot_service.getChangeSvc()
 
     # handle reloads through an http header
     # TODO: send this as a real header, rather than a tag
     def get_reload_time(self, request):
         if "reload" in request.args:
             try:
                 reload_time = int(request.args["reload"][0])
                 return max(reload_time, 15)
@@ -74,34 +77,31 @@ class GridStatusResource(HtmlResource):
 #        if p:
 #            self.title = "BuildBot: %s" % p
 #
     def build_td(self, request, build):
         if not build:
             return '<td class="build">&nbsp;</td>\n'
 
         if build.isFinished():
-            color = build.getColor()
-            if color == 'green': color = '#72ff75' # the "Buildbot Green"
-
             # get the text and annotate the first line with a link
             text = build.getText()
             if not text: text = [ "(no information)" ]
             if text == [ "build", "successful" ]: text = [ "OK" ]
         else:
-            color = 'yellow' # to match the yellow of the builder
             text = [ 'building' ]
 
         name = build.getBuilder().getName()
         number = build.getNumber()
         url = "builders/%s/builds/%d" % (name, number)
         text[0] = '<a href="%s">%s</a>' % (url, text[0])
         text = '<br />\n'.join(text)
+        class_ = build_get_class(build)
 
-        return '<td class="build" bgcolor="%s">%s</td>\n' % (color, text)
+        return '<td class="build %s">%s</td>\n' % (class_, text)
 
     def builder_td(self, request, builder):
         state, builds = builder.getState()
 
         # look for upcoming builds. We say the state is "waiting" if the
         # builder is otherwise idle and there is a scheduler which tells us a
         # build will be performed some time in the near future. TODO: this
         # functionality used to be in BuilderStatus.. maybe this code should
@@ -109,44 +109,33 @@ class GridStatusResource(HtmlResource):
         upcoming = []
         builderName = builder.getName()
         for s in self.getStatus(request).getSchedulers():
             if builderName in s.listBuilderNames():
                 upcoming.extend(s.getPendingBuildTimes())
         if state == "idle" and upcoming:
             state = "waiting"
 
-        if state == "building":
-            color = "yellow"
-        elif state == "offline":
-            color = "red"
-        elif state == "idle":
-            color = "white"
-        elif state == "waiting":
-            color = "yellow"
-        else:
-            color = "white"
-
         # TODO: for now, this pending/upcoming stuff is in the "current
         # activity" box, but really it should go into a "next activity" row
         # instead. The only times it should show up in "current activity" is
         # when the builder is otherwise idle.
 
         # are any builds pending? (waiting for a slave to be free)
         url = 'builders/%s/' % urllib.quote(builder.getName(), safe='')
         text = '<a href="%s">%s</a>' % (url, builder.getName())
         pbs = builder.getPendingBuilds()
         if state != 'idle' or pbs:
             if pbs:
                 text += "<br />(%s with %d pending)" % (state, len(pbs))
             else:
                 text += "<br />(%s)" % state
 
-        return  '<td valign="center" bgcolor="%s" class="builder">%s</td>\n' % \
-            (color, text)
+        return  '<td valign="center" class="builder %s">%s</td>\n' % \
+            (state, text)
 
     def stamp_td(self, stamp):
         text = stamp.getText()
         return '<td valign="bottom" class="sourcestamp">%s</td>\n' % \
             "<br />".join(text)
 
     def body(self, request):
         "This method builds the main waterfall display."
--- a/buildbot/status/web/logs.py
+++ b/buildbot/status/web/logs.py
@@ -80,16 +80,19 @@ class TextLog(Resource):
         data += '<a href="%s">(view as text)</a><br />\n' % texturl
         data += "<pre>\n"
         return data
 
     def content(self, entries):
         spanfmt = '<span class="%s">%s</span>'
         data = ""
         for type, entry in entries:
+            if type >= len(builder.ChunkTypes) or type < 0:
+                # non-std channel, don't display
+                continue
             if self.asText:
                 if type != builder.HEADER:
                     data += entry
             else:
                 data += spanfmt % (builder.ChunkTypes[type],
                                    html.escape(entry))
         return data
 
--- a/buildbot/status/web/slaves.py
+++ b/buildbot/status/web/slaves.py
@@ -1,15 +1,122 @@
 
-import time
-from buildbot.status.web.base import HtmlResource, abbreviate_age
+import time, urllib
+from twisted.python import log
+from twisted.web import html
+from twisted.web.util import Redirect
+
+from buildbot.status.web.base import HtmlResource, abbreviate_age, OneLineMixin, path_to_slave
+from buildbot import version, util
 
 # /buildslaves/$slavename
-class OneBuildSlaveResource(HtmlResource):
-    pass  # TODO
+class OneBuildSlaveResource(HtmlResource, OneLineMixin):
+    addSlash = False
+    def __init__(self, slavename):
+        HtmlResource.__init__(self)
+        self.slavename = slavename
+
+    def getTitle(self, req):
+        return "Buildbot: %s" % html.escape(self.slavename)
+
+    def getChild(self, path, req):
+        if path == "shutdown":
+            s = self.getStatus(req)
+            slave = s.getSlave(self.slavename)
+            slave.setGraceful(True)
+        return Redirect(path_to_slave(req, slave))
+
+    def body(self, req):
+        s = self.getStatus(req)
+        slave = s.getSlave(self.slavename)
+        my_builders = []
+        for bname in s.getBuilderNames():
+            b = s.getBuilder(bname)
+            for bs in b.getSlaves():
+                slavename = bs.getName()
+                if bs.getName() == self.slavename:
+                    my_builders.append(b)
+
+        # Current builds
+        current_builds = []
+        for b in my_builders:
+            for cb in b.getCurrentBuilds():
+                if cb.getSlavename() == self.slavename:
+                    current_builds.append(cb)
+
+        data = []
+
+        projectName = s.getProjectName()
+
+        data.append("<a href=\"%s\">%s</a>\n" % (self.path_to_root(req), projectName))
+
+        data.append("<h1>Build Slave: %s</h1>\n" % self.slavename)
+
+        shutdown_url = req.childLink("shutdown")
+
+        if not slave.isConnected():
+            data.append("<h2>NOT CONNECTED</h2>\n")
+        elif not slave.getGraceful():
+            data.append('''<form method="POST" action="%s">
+<input type="submit" value="Gracefully Shutdown">
+</form>''' % shutdown_url)
+        else:
+            data.append("Gracefully shutting down...\n")
+
+        if current_builds:
+            data.append("<h2>Currently building:</h2>\n")
+            data.append("<ul>\n")
+            for build in current_builds:
+                data.append("<li>%s</li>\n" % self.make_line(req, build, True))
+            data.append("</ul>\n")
+
+        else:
+            data.append("<h2>no current builds</h2>\n")
+
+        # Recent builds
+        data.append("<h2>Recent builds:</h2>\n")
+        data.append("<ul>\n")
+        n = 0
+        try:
+            max_builds = int(req.args.get('builds')[0])
+        except:
+            max_builds = 10
+        for build in s.generateFinishedBuilds(builders=[b.getName() for b in my_builders]):
+            if build.getSlavename() == self.slavename:
+                n += 1
+                data.append("<li>%s</li>\n" % self.make_line(req, build, True))
+                if n > max_builds:
+                    break
+        data.append("</ul>\n")
+
+        projectURL = s.getProjectURL()
+        projectName = s.getProjectName()
+        data.append('<hr /><div class="footer">\n')
+
+        welcomeurl = self.path_to_root(req) + "index.html"
+        data.append("[<a href=\"%s\">welcome</a>]\n" % welcomeurl)
+        data.append("<br />\n")
+
+        data.append('<a href="http://buildbot.sourceforge.net/">Buildbot</a>')
+        data.append("-%s " % version)
+        if projectName:
+            data.append("working for the ")
+            if projectURL:
+                data.append("<a href=\"%s\">%s</a> project." % (projectURL,
+                                                            projectName))
+            else:
+                data.append("%s project." % projectName)
+        data.append("<br />\n")
+        data.append("Page built: " +
+                 time.strftime("%a %d %b %Y %H:%M:%S",
+                               time.localtime(util.now()))
+                 + "\n")
+        data.append("</div>\n")
+
+        return "".join(data)
 
 # /buildslaves
 class BuildSlavesResource(HtmlResource):
     title = "BuildSlaves"
     addSlash = True
 
     def body(self, req):
         s = self.getStatus(req)
@@ -21,21 +128,21 @@ class BuildSlavesResource(HtmlResource):
             b = s.getBuilder(bname)
             for bs in b.getSlaves():
                 slavename = bs.getName()
                 if slavename not in used_by_builder:
                     used_by_builder[slavename] = []
                 used_by_builder[slavename].append(bname)
 
         data += "<ol>\n"
-        for name in s.getSlaveNames():
+        for name in util.naturalSort(s.getSlaveNames()):
             slave = s.getSlave(name)
             slave_status = s.botmaster.slaves[name].slave_status
             isBusy = len(slave_status.getRunningBuilds())
-            data += " <li>%s:\n" % name
+            data += " <li><a href=\"%s\">%s</a>:\n" % (req.childLink(urllib.quote(name,'')), name)
             data += " <ul>\n"
             builder_links = ['<a href="%s">%s</a>'
                              % (req.childLink("../builders/%s" % bname),bname)
                              for bname in used_by_builder.get(name, [])]
             if builder_links:
                 data += ("  <li>Used by Builders: %s</li>\n" %
                          ", ".join(builder_links))
             else:
@@ -64,8 +171,11 @@ class BuildSlavesResource(HtmlResource):
 
             data += " </ul>\n"
             data += " </li>\n"
             data += "\n"
 
         data += "</ol>\n"
 
         return data
+
+    def getChild(self, path, req):
+        return OneBuildSlaveResource(path)
--- a/buildbot/status/web/waterfall.py
+++ b/buildbot/status/web/waterfall.py
@@ -19,29 +19,28 @@ from buildbot.status.web.base import Box
 
 class CurrentBox(components.Adapter):
     # this provides the "current activity" box, just above the builder name
     implements(ICurrentBox)
 
     def formatETA(self, prefix, eta):
         if eta is None:
             return []
-        if eta < 0:
-            return ["Soon"]
-        eta_parts = []
+        if eta < 60:
+            return ["< 1 min"]
+        eta_parts = ["~"]
         eta_secs = eta
         if eta_secs > 3600:
             eta_parts.append("%d hrs" % (eta_secs / 3600))
             eta_secs %= 3600
         if eta_secs > 60:
             eta_parts.append("%d mins" % (eta_secs / 60))
             eta_secs %= 60
-        eta_parts.append("%d secs" % eta_secs)
-        abstime = time.strftime("%H:%M:%S", time.localtime(util.now()+eta))
-        return [prefix, ", ".join(eta_parts), "at %s" % abstime]
+        abstime = time.strftime("%H:%M", time.localtime(util.now()+eta))
+        return [prefix, " ".join(eta_parts), "at %s" % abstime]
 
     def getBox(self, status):
         # getState() returns offline, idle, or building
         state, builds = self.original.getState()
 
         # look for upcoming builds. We say the state is "waiting" if the
         # builder is otherwise idle and there is a scheduler which tells us a
         # build will be performed some time in the near future. TODO: this
@@ -51,100 +50,91 @@ class CurrentBox(components.Adapter):
         builderName = self.original.getName()
         for s in status.getSchedulers():
             if builderName in s.listBuilderNames():
                 upcoming.extend(s.getPendingBuildTimes())
         if state == "idle" and upcoming:
             state = "waiting"
 
         if state == "building":
-            color = "yellow"
             text = ["building"]
             if builds:
                 for b in builds:
                     eta = b.getETA()
                     text.extend(self.formatETA("ETA in", eta))
         elif state == "offline":
-            color = "red"
             text = ["offline"]
         elif state == "idle":
-            color = "white"
             text = ["idle"]
         elif state == "waiting":
-            color = "yellow"
             text = ["waiting"]
         else:
             # just in case I add a state and forget to update this
-            color = "white"
             text = [state]
 
         # TODO: for now, this pending/upcoming stuff is in the "current
         # activity" box, but really it should go into a "next activity" row
         # instead. The only times it should show up in "current activity" is
         # when the builder is otherwise idle.
 
         # are any builds pending? (waiting for a slave to be free)
         pbs = self.original.getPendingBuilds()
         if pbs:
             text.append("%d pending" % len(pbs))
         for t in upcoming:
             eta = t - util.now()
             text.extend(self.formatETA("next in", eta))
-        return Box(text, color=color, class_="Activity " + state)
+        return Box(text, class_="Activity " + state)
 
 components.registerAdapter(CurrentBox, builder.BuilderStatus, ICurrentBox)
 
 
 class BuildTopBox(components.Adapter):
     # this provides a per-builder box at the very top of the display,
     # showing the results of the most recent build
     implements(IBox)
 
     def getBox(self, req):
         assert interfaces.IBuilderStatus(self.original)
         branches = [b for b in req.args.get("branch", []) if b]
         builder = self.original
         builds = list(builder.generateFinishedBuilds(map_branches(branches),
                                                      num_builds=1))
         if not builds:
-            return Box(["none"], "white", class_="LastBuild")
+            return Box(["none"], class_="LastBuild")
         b = builds[0]
         name = b.getBuilder().getName()
         number = b.getNumber()
         url = path_to_build(req, b)
         text = b.getText()
         tests_failed = b.getSummaryStatistic('tests-failed', operator.add, 0)
         if tests_failed: text.extend(["Failed tests: %d" % tests_failed])
         # TODO: maybe add logs?
         # TODO: add link to the per-build page at 'url'
-        c = b.getColor()
         class_ = build_get_class(b)
-        return Box(text, c, class_="LastBuild %s" % class_)
+        return Box(text, class_="LastBuild %s" % class_)
 components.registerAdapter(BuildTopBox, builder.BuilderStatus, ITopBox)
 
 class BuildBox(components.Adapter):
     # this provides the yellow "starting line" box for each build
     implements(IBox)
 
     def getBox(self, req):
         b = self.original
         number = b.getNumber()
         url = path_to_build(req, b)
         reason = b.getReason()
         text = ('<a title="Reason: %s" href="%s">Build %d</a>'
                 % (html.escape(reason), url, number))
-        color = "yellow"
         class_ = "start"
         if b.isFinished() and not b.getSteps():
             # the steps have been pruned, so there won't be any indication
-            # of whether it succeeded or failed. Color the box red or green
-            # to show its status
-            color = b.getColor()
+            # of whether it succeeded or failed.
             class_ = build_get_class(b)
-        return Box([text], color=color, class_="BuildStep " + class_)
+        return Box([text], class_="BuildStep " + class_)
 components.registerAdapter(BuildBox, builder.BuildStatus, IBox)
 
 class StepBox(components.Adapter):
     implements(IBox)
 
     def getBox(self, req):
         urlbase = path_to_step(req, self.original)
         text = self.original.getText()
@@ -160,48 +150,42 @@ class StepBox(components.Adapter):
                 text.append("<a href=\"%s\">%s</a>" % (url, html.escape(name)))
             else:
                 text.append(html.escape(name))
         urls = self.original.getURLs()
         ex_url_class = "BuildStep external"
         for name, target in urls.items():
             text.append('[<a href="%s" class="%s">%s</a>]' %
                         (target, ex_url_class, html.escape(name)))
-        color = self.original.getColor()
         class_ = "BuildStep " + build_get_class(self.original)
-        return Box(text, color, class_=class_)
+        return Box(text, class_=class_)
 components.registerAdapter(StepBox, builder.BuildStepStatus, IBox)
 
 
 class EventBox(components.Adapter):
     implements(IBox)
 
     def getBox(self, req):
         text = self.original.getText()
-        color = self.original.getColor()
         class_ = "Event"
-        if color:
-            class_ += " " + color
-        return Box(text, color, class_=class_)
+        return Box(text, class_=class_)
 components.registerAdapter(EventBox, builder.Event, IBox)
         
 
 class Spacer:
     implements(interfaces.IStatusEvent)
 
     def __init__(self, start, finish):
         self.started = start
         self.finished = finish
 
     def getTimes(self):
         return (self.started, self.finished)
     def getText(self):
         return []
-    def getColor(self):
-        return None
 
 class SpacerBox(components.Adapter):
     implements(IBox)
 
     def getBox(self, req):
         #b = Box(["spacer"], "white")
         b = Box([])
         b.spacer = True
@@ -421,17 +405,17 @@ class WaterfallStatusResource(HtmlResour
         p = status.getProjectName()
         if p:
             return "BuildBot: %s" % p
         else:
             return "BuildBot"
 
     def getChangemaster(self, request):
         # TODO: this wants to go away, access it through IStatus
-        return request.site.buildbot_service.parent.change_svc
+        return request.site.buildbot_service.getChangeSvc()
 
     def get_reload_time(self, request):
         if "reload" in request.args:
             try:
                 reload_time = int(request.args["reload"][0])
                 return max(reload_time, 15)
             except ValueError:
                 pass
@@ -607,41 +591,39 @@ class WaterfallStatusResource(HtmlResour
         data += '<table border="0" cellspacing="0">\n'
         names = map(lambda builder: builder.name, builders)
 
         # the top row is two blank spaces, then the top-level status boxes
         data += " <tr>\n"
         data += td("", colspan=2)
         for b in builders:
             text = ""
-            color = "#ca88f7"
             state, builds = b.getState()
             if state != "offline":
                 text += "%s<br />\n" % state #b.getCurrentBig().text[0]
             else:
                 text += "OFFLINE<br />\n"
-                color = "#ffe0e0"
-            data += td(text, align="center", bgcolor=color)
+            data += td(text, align="center")
 
         # the next row has the column headers: time, changes, builder names
         data += " <tr>\n"
         data += td("Time", align="center")
         data += td("Changes", align="center")
         for name in names:
             data += td('<a href="%s">%s</a>' %
                        (request.childLink("../" + urllib.quote(name)), name),
                        align="center")
         data += " </tr>\n"
 
         # all further rows involve timestamps, commit events, and build events
         data += " <tr>\n"
         data += td("04:00", align="bottom")
         data += td("fred", align="center")
         for name in names:
-            data += td("stuff", align="center", bgcolor="red")
+            data += td("stuff", align="center")
         data += " </tr>\n"
 
         data += "</table>\n"
         return data
     
     def buildGrid(self, request, builders):
         debug = False
         # TODO: see if we can use a cached copy
@@ -789,19 +771,18 @@ class WaterfallStatusResource(HtmlResour
             row = eventGrid[r]
             assert(len(row) == len(sourceNames))
             for c in range(0, len(row)):
                 if row[c]:
                     data += "<b>%s</b><br />\n" % sourceNames[c]
                     for e in row[c]:
                         log.msg("Event", r, c, sourceNames[c], e.getText())
                         lognames = [loog.getName() for loog in e.getLogs()]
-                        data += "%s: %s: %s %s<br />" % (e.getText(),
+                        data += "%s: %s: %s<br />" % (e.getText(),
                                                          e.getTimes()[0],
-                                                         e.getColor(),
                                                          lognames)
                 else:
                     data += "<b>%s</b> [none]<br />\n" % sourceNames[c]
         return data
     
     def phase1(self, request, sourceNames, timestamps, eventGrid,
                sourceEvents):
         # phase1 rendering: table, but boxes do not overlap
--- a/buildbot/status/web/xmlrpc.py
+++ b/buildbot/status/web/xmlrpc.py
@@ -158,22 +158,33 @@ class XMLRPCServer(xmlrpc.XMLRPC):
     def xmlrpc_getBuild(self, builder_name, build_number):
         """Return information about a specific build.
 
         """
         builder = self.status.getBuilder(builder_name)
         build = builder.getBuild(build_number)
         info = {}
         info['builder_name'] = builder.getName()
-        info['url'] = self.status.getURLForThing(build)
+        info['url'] = self.status.getURLForThing(build) or ''
         info['reason'] = build.getReason()
         info['slavename'] = build.getSlavename()
         info['results'] = build.getResults()
         info['text'] = build.getText()
+        # Added to help out requests for build -N
+        info['number'] = build.number
         ss = build.getSourceStamp()
+        branch = ss.branch
+        if branch is None:
+            branch = ""
+        info['branch'] = str(branch)
+        try:
+            revision = str(build.getProperty("got_revision"))
+        except KeyError:
+            revision = ""
+        info['revision'] = str(revision)
         info['start'], info['end'] = build.getTimes()
 
         info_steps = []
         for s in build.getSteps():
             stepinfo = {}
             stepinfo['name'] = s.getName()
             stepinfo['start'], stepinfo['end'] = s.getTimes()
             stepinfo['results'] = s.getResults()
@@ -183,11 +194,10 @@ class XMLRPCServer(xmlrpc.XMLRPC):
         info_logs = []
         for l in build.getLogs():
             loginfo = {}
             loginfo['name'] = l.getStep().getName() + "/" + l.getName()
             #loginfo['text'] = l.getText()
             loginfo['text'] = "HUGE"
             info_logs.append(loginfo)
         info['logs'] = info_logs
-
         return info
 
--- a/buildbot/status/words.py
+++ b/buildbot/status/words.py
@@ -13,16 +13,18 @@ from twisted.application import internet
 from buildbot import interfaces, util
 from buildbot import version
 from buildbot.sourcestamp import SourceStamp
 from buildbot.process.base import BuildRequest
 from buildbot.status import base
 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
 from buildbot.scripts.runner import ForceOptions
 
+from string import join, capitalize, lower
+
 class UsageError(ValueError):
     def __init__(self, string = "Invalid usage", *more):
         ValueError.__init__(self, string, *more)
 
 class IrcBuildRequest:
     hasStarted = False
     timer = None
 
@@ -44,17 +46,17 @@ class IrcBuildRequest:
         s = c.getStatus()
         eta = s.getETA()
         response = "build #%d forced" % s.getNumber()
         if eta is not None:
             response = "build forced [ETA %s]" % self.parent.convertTime(eta)
         self.parent.send(response)
         self.parent.send("I'll give a shout when the build finishes")
         d = s.waitUntilFinished()
-        d.addCallback(self.parent.buildFinished)
+        d.addCallback(self.parent.watchedBuildFinished)
 
 
 class Contact:
     """I hold the state for a single user's interaction with the buildbot.
 
     This base class provides all the basic behavior (the queries and
     responses). Subclasses for each channel type (IRC, different IM
     protocols) are expected to provide the lower-level send/receive methods.
@@ -63,16 +65,17 @@ class Contact:
     with the buildbot. There will be an additional instance for each
     'broadcast contact' (chat rooms, IRC channels as a whole).
     """
 
     def __init__(self, channel):
         self.channel = channel
         self.notify_events = {}
         self.subscribed = 0
+	self.add_notification_events(channel.notify_events)
 
     silly = {
         "What happen ?": "Somebody set up us the bomb.",
         "It's You !!": ["How are you gentlemen !!",
                         "All your base are belong to us.",
                         "You are on the way to destruction."],
         "What you say !!": ["You have no chance to survive make your time.",
                             "HA HA HA HA ...."],
@@ -163,17 +166,17 @@ class Contact:
             builders = self.getAllBuilders()
             for b in builders:
                 self.emit_status(b.name)
             return
         self.emit_status(which)
     command_STATUS.usage = "status [<which>] - List status of a builder (or all builders)"
 
     def validate_notification_event(self, event):
-        if not re.compile("^(started|finished|success|failed|exception)$").match(event):
+        if not re.compile("^(started|finished|success|failure|exception|warnings|(success|warnings|exception|failure)To(Failure|Success|Warnings|Exception))$").match(event):
             raise UsageError("try 'notify on|off <EVENT>'")
 
     def list_notified_events(self):
         self.send( "The following events are being notified: %r" % self.notify_events.keys() )
 
     def notify_for(self, *events):
         for event in events:
             if self.notify_events.has_key(event):
@@ -188,24 +191,33 @@ class Contact:
         self.channel.status.unsubscribe(self)
         self.subscribed = 0
 
     def add_notification_events(self, events):
         for event in events:
             self.validate_notification_event(event)
             self.notify_events[event] = 1
 
+            if not self.subscribed:
+                self.subscribe_to_build_events()
+
     def remove_notification_events(self, events):
         for event in events:
             self.validate_notification_event(event)
             del self.notify_events[event]
 
+            if len(self.notify_events) == 0 and self.subscribed:
+                self.unsubscribe_from_build_events()
+
     def remove_all_notification_events(self):
         self.notify_events = {}
 
+        if self.subscribed:
+            self.unsubscribe_from_build_events()
+
     def command_NOTIFY(self, args, who):
         args = args.split()
 
         if not args:
             raise UsageError("try 'notify on|off|list <EVENT>'")
         action = args.pop(0)
         events = args
 
@@ -225,23 +237,17 @@ class Contact:
 
         elif action == "list":
             self.list_notified_events()
             return
 
         else:
             raise UsageError("try 'notify on|off <EVENT>'")
 
-        if len(self.notify_events) > 0 and not self.subscribed:
-            self.subscribe_to_build_events()
-
-        elif len(self.notify_events) == 0 and self.subscribed:
-            self.unsubscribe_from_build_events()
-
-    command_NOTIFY.usage = "notify on|off|list [<EVENT>] ... - Notify me about build events.  event should be one or more of: 'started', 'finished', 'failed', 'success', 'exception', 'successToFailed', 'failedToSuccess'"
+    command_NOTIFY.usage = "notify on|off|list [<EVENT>] ... - Notify me about build events.  event should be one or more of: 'started', 'finished', 'failure', 'success', 'exception' or 'xToY' (where x and Y are one of success, warnings, failure, exception, but Y is capitalized)"
 
     def command_WATCH(self, args, who):
         args = args.split()
         if len(args) != 1:
             raise UsageError("try 'watch <builder>'")
         which = args[0]
         b = self.getBuilder(which)
         builds = b.getCurrentBuilds()
@@ -266,16 +272,20 @@ class Contact:
 
     def builderAdded(self, builderName, builder):
         log.msg('[Contact] Builder %s added' % (builder))
         builder.subscribe(self)
 
     def builderChangedState(self, builderName, state):
         log.msg('[Contact] Builder %s changed state to %s' % (builderName, state))
 
+    def requestSubmitted(self, brstatus):
+        log.msg('[Contact] BuildRequest for %s submiitted to Builder %s' % 
+            (brstatus.getSourceStamp(), brstatus.builderName))
+
     def builderRemoved(self, builderName):
         log.msg('[Contact] Builder %s removed' % (builderName))
 
     def buildStarted(self, builderName, build):
         builder = build.getBuilder()
         log.msg('[Contact] Builder %r in category %s started' % (builder, builder.category))
 
         # only notify about builders we are interested in
@@ -305,17 +315,17 @@ class Contact:
             WARNINGS: "Warnings",
             FAILURE: "Failure",
             EXCEPTION: "Exception",
             }
 
         # only notify about builders we are interested in
         log.msg('[Contact] builder %r in category %s finished' % (builder, builder.category))
 
-        if not self.notify_for('finished', 'failed', 'success', 'exception', 'failedToSuccess', 'successToFailed'):
+        if self.notify_for('started'):
             return
 
         if (self.channel.categories != None and
             builder.category not in self.channel.categories):
             return
 
         results = build.getResults()
 
@@ -323,29 +333,30 @@ class Contact:
             (build.getNumber(),
              builder.getName(),
              results_descriptions.get(results, "??"))
         r += " [%s]" % " ".join(build.getText())
         buildurl = self.channel.status.getURLForThing(build)
         if buildurl:
             r += "  Build details are at %s" % buildurl
 
-        if (self.notify_for('finished')) or \
-           (self.notify_for('success') and results == SUCCESS) or \
-           (self.notify_for('failed') and results == FAILURE) or \
-           (self.notify_for('exception') and results == EXCEPTION):
+        if self.notify_for('finished') or self.notify_for(lower(results_descriptions.get(results))):
             self.send(r)
             return
 
         prevBuild = build.getPreviousBuild()
         if prevBuild:
-            prevResult = prevBuild.getResult()
+            prevResult = prevBuild.getResults()
 
-            if (self.notify_for('failureToSuccess') and prevResult == FAILURE and results == SUCCESS) or \
-               (self.notify_for('successToFailure') and prevResult == SUCCESS and results == FAILURE):
+            required_notification_control_string = join((lower(results_descriptions.get(prevResult)), \
+                                                             'To', \
+                                                             capitalize(results_descriptions.get(results))), \
+                                                            '')
+
+            if (self.notify_for(required_notification_control_string)):
                 self.send(r)
 
     def watchedBuildFinished(self, b):
         results = {SUCCESS: "Success",
                    WARNINGS: "Warnings",
                    FAILURE: "Failure",
                    EXCEPTION: "Exception",
                    }
@@ -450,18 +461,18 @@ class Contact:
         b = self.getBuilder(which)
         str = "%s: " % which
         state, builds = b.getState()
         str += state
         if state == "idle":
             last = b.getLastFinishedBuild()
             if last:
                 start,finished = last.getTimes()
-                str += ", last build %s secs ago: %s" % \
-                       (int(util.now() - finished), " ".join(last.getText()))
+                str += ", last build %s ago: %s" % \
+                        (self.convertTime(int(util.now() - finished)), " ".join(last.getText()))
         if state == "building":
             t = []
             for build in builds:
                 step = build.getCurrentStep()
                 if step:
                     s = "(%s)" % " ".join(step.getText())
                 else:
                     s = "(no current step)"
@@ -473,17 +484,17 @@ class Contact:
         self.send(str)
 
     def emit_last(self, which):
         last = self.getBuilder(which).getLastFinishedBuild()
         if not last:
             str = "(no builds run since last restart)"
         else:
             start,finish = last.getTimes()
-            str = "%s secs ago: " % (int(util.now() - finish))
+            str = "%s ago: " % (self.convertTime(int(util.now() - finish)))
             str += " ".join(last.getText())
         self.send("last build [%s]: %s" % (which, str))
 
     def command_LAST(self, args, who):
         args = args.split()
         if len(args) == 0:
             which = "all"
         elif len(args) == 1:
@@ -575,19 +586,19 @@ class IRCContact(Contact):
     def describeUser(self, user):
         if self.dest[0] == "#":
             return "IRC user <%s> on channel %s" % (user, self.dest)
         return "IRC user <%s> (privmsg)" % user
 
     # userJoined(self, user, channel)
 
     def send(self, message):
-        self.channel.msg(self.dest, message)
+        self.channel.msg(self.dest, message.encode("ascii", "replace"))
     def act(self, action):
-        self.channel.me(self.dest, action)
+        self.channel.me(self.dest, action.encode("ascii", "replace"))
 
     def command_JOIN(self, args, who):
         args = args.split()
         to_join = args[0]
         self.channel.join(to_join)
         self.send("Joined %s" % to_join)
     command_JOIN.usage = "join channel - Join another channel"
 
@@ -649,33 +660,34 @@ class IChannel(Interface):
     more Contacts associated with it.
     """
 
 class IrcStatusBot(irc.IRCClient):
     """I represent the buildbot to an IRC server.
     """
     implements(IChannel)
 
-    def __init__(self, nickname, password, channels, status, categories):
+    def __init__(self, nickname, password, channels, status, categories, notify_events):
         """
         @type  nickname: string
         @param nickname: the nickname by which this bot should be known
         @type  password: string
         @param password: the password to use for identifying with Nickserv
         @type  channels: list of strings
         @param channels: the bot will maintain a presence in these channels
         @type  status: L{buildbot.status.builder.Status}
         @param status: the build master's Status object, through which the
                        bot retrieves all status information
         """
         self.nickname = nickname
         self.channels = channels
         self.password = password
         self.status = status
         self.categories = categories
+        self.notify_events = notify_events
         self.counter = 0
         self.hasQuit = 0
         self.contacts = {}
 
     def addContact(self, name, contact):
         self.contacts[name] = contact
 
     def getContact(self, name):
@@ -760,38 +772,39 @@ class ThrottledClientFactory(protocol.Cl
 class IrcStatusFactory(ThrottledClientFactory):
     protocol = IrcStatusBot
 
     status = None
     control = None
     shuttingDown = False
     p = None
 
-    def __init__(self, nickname, password, channels, categories):
+    def __init__(self, nickname, password, channels, categories, notify_events):
         #ThrottledClientFactory.__init__(self) # doesn't exist
         self.status = None
         self.nickname = nickname
         self.password = password
         self.channels = channels
         self.categories = categories
+	self.notify_events = notify_events
 
     def __getstate__(self):
         d = self.__dict__.copy()
         del d['p']
         return d
 
     def shutdown(self):
         self.shuttingDown = True
         if self.p:
             self.p.quit("buildmaster reconfigured: bot disconnecting")
 
     def buildProtocol(self, address):
         p = self.protocol(self.nickname, self.password,
                           self.channels, self.status,
-                          self.categories)
+                          self.categories, self.notify_events)
         p.factory = self
         p.status = self.status
         p.control = self.control
         self.p = p
         return p
 
     # TODO: I think a shutdown that occurs while the connection is being
     # established will make this explode
@@ -814,33 +827,34 @@ class IRC(base.StatusReceiverMultiServic
     connect to a single IRC server and am known by a single nickname on that
     server, however I can join multiple channels."""
 
     compare_attrs = ["host", "port", "nick", "password",
                      "channels", "allowForce",
                      "categories"]
 
     def __init__(self, host, nick, channels, port=6667, allowForce=True,
-                 categories=None, password=None):
+                 categories=None, password=None, notify_events={}):
         base.StatusReceiverMultiService.__init__(self)
 
         assert allowForce in (True, False) # TODO: implement others
 
         # need to stash these so we can detect changes later
         self.host = host
         self.port = port
         self.nick = nick
         self.channels = channels
         self.password = password
         self.allowForce = allowForce
         self.categories = categories
+	self.notify_events = notify_events
 
         # need to stash the factory so we can give it the status object
         self.f = IrcStatusFactory(self.nick, self.password,
-                                  self.channels, self.categories)
+                                  self.channels, self.categories, self.notify_events)
 
         c = internet.TCPClient(host, port, self.f)
         c.setServiceParent(self)
 
     def setServiceParent(self, parent):
         base.StatusReceiverMultiService.setServiceParent(self, parent)
         self.f.status = parent.getStatus()
         if self.allowForce:
--- a/buildbot/steps/dummy.py
+++ b/buildbot/steps/dummy.py
@@ -7,67 +7,64 @@ from buildbot.status.builder import SUCC
 # these classes are used internally by buildbot unit tests
 
 class Dummy(BuildStep):
     """I am a dummy no-op step, which runs entirely on the master, and simply
     waits 5 seconds before finishing with SUCCESS
     """
 
     haltOnFailure = True
+    flunkOnFailure = True
     name = "dummy"
 
     def __init__(self, timeout=5, **kwargs):
         """
         @type  timeout: int
         @param timeout: the number of seconds to delay before completing
         """
         BuildStep.__init__(self, **kwargs)
         self.addFactoryArguments(timeout=timeout)
         self.timeout = timeout
         self.timer = None
 
     def start(self):
-        self.step_status.setColor("yellow")
         self.step_status.setText(["delay", "%s secs" % self.timeout])
         self.timer = reactor.callLater(self.timeout, self.done)
 
     def interrupt(self, reason):
         if self.timer:
             self.timer.cancel()
             self.timer = None
-            self.step_status.setColor("red")
             self.step_status.setText(["delay", "interrupted"])
             self.finished(FAILURE)
 
     def done(self):
-        self.step_status.setColor("green")
         self.finished(SUCCESS)
 
 class FailingDummy(Dummy):
     """I am a dummy no-op step that 'runs' master-side and finishes (with a
     FAILURE status) after 5 seconds."""
 
     name = "failing dummy"
 
     def start(self):
-        self.step_status.setColor("yellow")
         self.step_status.setText(["boom", "%s secs" % self.timeout])
         self.timer = reactor.callLater(self.timeout, self.done)
 
     def done(self):
-        self.step_status.setColor("red")
         self.finished(FAILURE)
 
 class RemoteDummy(LoggingBuildStep):
     """I am a dummy no-op step that runs on the remote side and
     simply waits 5 seconds before completing with success.
     See L{buildbot.slave.commands.DummyCommand}
     """
 
     haltOnFailure = True
+    flunkOnFailure = True
     name = "remote dummy"
 
     def __init__(self, timeout=5, **kwargs):
         """
         @type  timeout: int
         @param timeout: the number of seconds to delay
         """
         LoggingBuildStep.__init__(self, **kwargs)
--- a/buildbot/steps/maxq.py
+++ b/buildbot/steps/maxq.py
@@ -29,18 +29,16 @@ class MaxQ(ShellCommand):
 
         if self.failures:
             result = (FAILURE, [str(self.failures), 'maxq', 'failures'])
 
         return self.stepComplete(result)
 
     def finishStatus(self, result):
         if self.failures:
-            color = "red"
             text = ["maxq", "failed"]
         else:
-            color = "green"
             text = ['maxq', 'tests']
-        self.updateCurrentActivity(color=color, text=text)
+        self.updateCurrentActivity(text=text)
         self.finishStatusSummary()
         self.finishCurrentActivity()
 
 
--- a/buildbot/steps/python.py
+++ b/buildbot/steps/python.py
@@ -1,11 +1,12 @@
 
 from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS
 from buildbot.steps.shell import ShellCommand
+import re
 
 try:
     import cStringIO
     StringIO = cStringIO.StringIO
 except ImportError:
     from StringIO import StringIO
 
 
@@ -105,8 +106,82 @@ class PyFlakes(ShellCommand):
             return FAILURE
         for m in self.flunkingIssues:
             if self.getProperty("pyflakes-%s" % m):
                 return FAILURE
         if self.getProperty("pyflakes-total"):
             return WARNINGS
         return SUCCESS
 
+class PyLint(ShellCommand):
+    '''A command that knows about pylint output.
+    It's a good idea to add --output-format=parseable to your
+    command, since it includes the filename in the message.
+    '''
+    name = "pylint"
+    description = ["running", "pylint"]
+    descriptionDone = ["pylint"]
+
+    # Using the default text output, the message format is :
+    # MESSAGE_TYPE: LINE_NUM:[OBJECT:] MESSAGE
+    # with --output-format=parseable it is: (the outer brackets are literal)
+    # FILE_NAME:LINE_NUM: [MESSAGE_TYPE[, OBJECT]] MESSAGE
+    # message type consists of the type char and 4 digits
+    # The message types:
+
+    MESSAGES = {
+            'C': "convention", # for programming standard violation
+            'R': "refactor", # for bad code smell
+            'W': "warning", # for python specific problems
+            'E': "error", # for much probably bugs in the code
+            'F': "fatal", # error prevented pylint from further processing.
+            'I': "info",
+        }
+
+    flunkingIssues = ["F", "E"] # msg categories that cause FAILURE
+
+    _re_groupname = 'errtype'
+    _msgtypes_re_str = '(?P<%s>[%s])' % (_re_groupname, ''.join(MESSAGES.keys()))
+    _default_line_re = re.compile(r'%s\d{4}: *\d+:.+' % _msgtypes_re_str)
+    _parseable_line_re = re.compile(r'[^:]+:\d+: \[%s\d{4}[,\]] .+' % _msgtypes_re_str)
+
+    def createSummary(self, log):
+        counts = {}
+        summaries = {}
+        for m in self.MESSAGES:
+            counts[m] = 0
+            summaries[m] = []
+
+        line_re = None # decide after first match
+        for line in StringIO(log.getText()).readlines():
+            if not line_re:
+                # need to test both and then decide on one
+                if self._parseable_line_re.match(line):
+                    line_re = self._parseable_line_re
+                elif self._default_line_re.match(line):
+                    line_re = self._default_line_re
+                else: # no match yet
+                    continue
+            mo = line_re.match(line)
+            if mo:
+                msgtype = mo.group(self._re_groupname)
+                assert msgtype in self.MESSAGES
+            summaries[msgtype].append(line)
+            counts[msgtype] += 1
+
+        self.descriptionDone = self.descriptionDone[:]
+        for msg, fullmsg in self.MESSAGES.items():
+            if counts[msg]:
+                self.descriptionDone.append("%s=%d" % (fullmsg, counts[msg]))
+                self.addCompleteLog(fullmsg, "".join(summaries[msg]))
+            self.setProperty("pylint-%s" % fullmsg, counts[msg])
+        self.setProperty("pylint-total", sum(counts.values()))
+
+    def evaluateCommand(self, cmd):
+        if cmd.rc != 0:
+            return FAILURE
+        for msg in self.flunkingIssues:
+            if self.getProperty("pylint-%s" % self.MESSAGES[msg]):
+                return FAILURE
+        if self.getProperty("pylint-total"):
+            return WARNINGS
+        return SUCCESS
+
--- a/buildbot/steps/python_twisted.py
+++ b/buildbot/steps/python_twisted.py
@@ -181,22 +181,16 @@ class Trial(ShellCommand):
     test cases are located. The most common way of doing this is with a
     module name. For petmail, I would run 'trial petmail.test' and it would
     locate all the test_*.py files under petmail/test/, running every test
     case it could find in them. Unlike the unittest.py that comes with
     Python, you do not run the test_foo.py as a script; you always let trial
     do the importing and running. The 'tests' parameter controls which tests
     trial will run: it can be a string or a list of strings.
 
-    You can also use a higher-level module name and pass the --recursive flag
-    to trial: this will search recursively within the named module to find
-    all test cases. For large multiple-test-directory projects like Twisted,
-    this means you can avoid specifying all the test directories explicitly.
-    Something like 'trial --recursive twisted' will pick up everything.
-
     To find these test cases, you must set a PYTHONPATH that allows something
     like 'import petmail.test' to work. For packages that don't use a
     separate top-level 'lib' directory, PYTHONPATH=. will work, and will use
     the test cases (and the code they are testing) in-place.
     PYTHONPATH=build/lib or PYTHONPATH=build/lib.$ARCH are also useful when
     you do a'setup.py build' step first. The 'testpath' attribute of this
     class controls what PYTHONPATH= is set to.
 
@@ -684,22 +678,16 @@ class ProcessDocs(ShellCommand):
     descriptionDone = ["docs"]
     # TODO: track output and time
 
     def __init__(self, **kwargs):
         """
         @type    workdir: string
         @keyword workdir: the workdir to start from: must be the base of the
                           Twisted tree
-
-        @type    results: triple of (int, int, string)
-        @keyword results: [rc, warnings, output]
-                          - rc==0 if all files were converted successfully.
-                          - warnings is a count of hlint warnings. 
-                          - output is the verbose output of the command.
         """
         ShellCommand.__init__(self, **kwargs)
 
     def createSummary(self, log):
         output = log.getText()
         # hlint warnings are of the format: 'WARNING: file:line:col: stuff
         # latex warnings start with "WARNING: LaTeX Warning: stuff", but
         # sometimes wrap around to a second line.
@@ -753,20 +741,16 @@ class BuildDebs(ShellCommand):
     description = ["building", "debs"]
     descriptionDone = ["debs"]
 
     def __init__(self, **kwargs):
         """
         @type    workdir: string
         @keyword workdir: the workdir to start from (must be the base of the
                           Twisted tree)
-        @type    results: double of [int, string]
-        @keyword results: [rc, output].
-                          - rc == 0 if all .debs were created successfully
-                          - output: string with any errors or warnings
         """
         ShellCommand.__init__(self, **kwargs)
 
     def commandComplete(self, cmd):
         errors, warnings = 0, 0
         output = cmd.logs['stdio'].getText()
         summary = ""
         sio = StringIO.StringIO(output)
--- a/buildbot/steps/shell.py
+++ b/buildbot/steps/shell.py
@@ -51,47 +51,49 @@ class ShellCommand(LoggingBuildStep):
 
     # override this on a specific ShellCommand if you want to let it fail
     # without dooming the entire build to a status of FAILURE
     flunkOnFailure = True
 
     def __init__(self, workdir=None,
                  description=None, descriptionDone=None,
                  command=None,
+                 usePTY="slave-config",
                  **kwargs):
         # most of our arguments get passed through to the RemoteShellCommand
         # that we create, but first strip out the ones that we pass to
         # BuildStep (like haltOnFailure and friends), and a couple that we
         # consume ourselves.
 
         if description:
             self.description = description
         if isinstance(self.description, str):
             self.description = [self.description]
         if descriptionDone:
             self.descriptionDone = descriptionDone
         if isinstance(self.descriptionDone, str):
             self.descriptionDone = [self.descriptionDone]
         if command:
-            self.command = command
+            self.setCommand(command)
 
         # pull out the ones that LoggingBuildStep wants, then upcall
         buildstep_kwargs = {}
         for k in kwargs.keys()[:]:
             if k in self.__class__.parms:
                 buildstep_kwargs[k] = kwargs[k]
                 del kwargs[k]
         LoggingBuildStep.__init__(self, **buildstep_kwargs)
         self.addFactoryArguments(workdir=workdir,
                                  description=description,
                                  descriptionDone=descriptionDone,
                                  command=command)
 
         # everything left over goes to the RemoteShellCommand
         kwargs['workdir'] = workdir # including a copy of 'workdir'
+        kwargs['usePTY'] = usePTY
         self.remote_kwargs = kwargs
         # we need to stash the RemoteShellCommand's args too
         self.addFactoryArguments(**kwargs)
 
     def setDefaultWorkdir(self, workdir):
         rkw = self.remote_kwargs
         rkw['workdir'] = rkw['workdir'] or workdir
 
@@ -111,46 +113,49 @@ class ShellCommand(LoggingBuildStep):
                      imperfect-tense verb is appropriate ('compiling',
                      'testing', ...) C{done=True} is used when the command
                      has finished, and the default getText() method adds some
                      text, so a simple noun is appropriate ('compile',
                      'tests' ...)
         """
 
         if done and self.descriptionDone is not None:
-            return self.descriptionDone
+            return list(self.descriptionDone)
         if self.description is not None:
-            return self.description
+            return list(self.description)
 
         properties = self.build.getProperties()
         words = self.command
         if isinstance(words, (str, unicode)):
             words = words.split()
         # render() each word to handle WithProperties objects
         words = properties.render(words)
         if len(words) < 1:
             return ["???"]
         if len(words) == 1:
             return ["'%s'" % words[0]]
         if len(words) == 2:
             return ["'%s" % words[0], "%s'" % words[1]]
         return ["'%s" % words[0], "%s" % words[1], "...'"]
 
     def setupEnvironment(self, cmd):
-        # XXX is this used? documented? replaced by properties?
-        # merge in anything from Build.slaveEnvironment . Earlier steps
-        # (perhaps ones which compile libraries or sub-projects that need to
-        # be referenced by later steps) can add keys to
-        # self.build.slaveEnvironment to affect later steps.
+        # merge in anything from Build.slaveEnvironment
+        # This can be set from a Builder-level environment, or from earlier
+        # BuildSteps. The latter method is deprecated and superceded by
+        # BuildProperties.
+        # Environment variables passed in by a BuildStep override
+        # those passed in at the Builder level.
         properties = self.build.getProperties()
         slaveEnv = self.build.slaveEnvironment
         if slaveEnv:
             if cmd.args['env'] is None:
                 cmd.args['env'] = {}
-            cmd.args['env'].update(properties.render(slaveEnv))
+            fullSlaveEnv = slaveEnv.copy()
+            fullSlaveEnv.update(cmd.args['env'])
+            cmd.args['env'] = properties.render(fullSlaveEnv)
             # note that each RemoteShellCommand gets its own copy of the
             # dictionary, so we shouldn't be affecting anyone but ourselves.
 
     def checkForOldSlaveAndLogfiles(self):
         if not self.logfiles:
             return # doesn't matter
         if not self.slaveVersionIsOlderThan("shell", "2.1"):
             return # slave is new enough
@@ -172,25 +177,34 @@ class ShellCommand(LoggingBuildStep):
         self.logfiles = {}
 
     def start(self):
         # this block is specific to ShellCommands. subclasses that don't need
         # to set up an argv array, an environment, or extra logfiles= (like
         # the Source subclasses) can just skip straight to startCommand()
         properties = self.build.getProperties()
 
+        warnings = []
+
         # create the actual RemoteShellCommand instance now
         kwargs = properties.render(self.remote_kwargs)
         kwargs['command'] = properties.render(self.command)
         kwargs['logfiles'] = self.logfiles
+
+        # check for the usePTY flag
+        if kwargs.has_key('usePTY') and kwargs['usePTY'] != 'slave-config':
+            slavever = self.slaveVersion("shell", "old")
+            if self.slaveVersionIsOlderThan("svn", "2.7"):
+                warnings.append("NOTE: slave does not allow master to override usePTY\n")
+
         cmd = RemoteShellCommand(**kwargs)
         self.setupEnvironment(cmd)
         self.checkForOldSlaveAndLogfiles()
 
-        self.startCommand(cmd)
+        self.startCommand(cmd, warnings)
 
 
 
 class TreeSize(ShellCommand):
     name = "treesize"
     command = ["du", "-s", "-k", "."]
     kib = None
 
@@ -267,39 +281,36 @@ class SetProperty(ShellCommand):
             return [ "set props:" ] + self.property_changes.keys()
         else:
             return [ "no change" ]
 
 class Configure(ShellCommand):
 
     name = "configure"
     haltOnFailure = 1
+    flunkOnFailure = 1
     description = ["configuring"]
     descriptionDone = ["configure"]
     command = ["./configure"]
 
 class WarningCountingShellCommand(ShellCommand):
     warnCount = 0
     warningPattern = '.*warning[: ].*'
 
-    def __init__(self, **kwargs):
+    def __init__(self, warningPattern=None, **kwargs):
         # See if we've been given a regular expression to use to match
         # warnings. If not, use a default that assumes any line with "warning"
         # present is a warning. This may lead to false positives in some cases.
-        wp = None
-        if kwargs.has_key('warningPattern'):
-            wp = kwargs['warningPattern']
-            del kwargs['warningPattern']
-            self.warningPattern = wp
+        if warningPattern:
+            self.warningPattern = warningPattern
 
         # And upcall to let the base class do its work
         ShellCommand.__init__(self, **kwargs)
 
-        if wp:
-            self.addFactoryArguments(warningPattern=wp)
+        self.addFactoryArguments(warningPattern=warningPattern)
 
     def createSummary(self, log):
         self.warnCount = 0
 
         # Now compile a regular expression from whichever warning pattern we're
         # using
         if not self.warningPattern:
             return
@@ -341,29 +352,29 @@ class WarningCountingShellCommand(ShellC
             return WARNINGS
         return SUCCESS
 
 
 class Compile(WarningCountingShellCommand):
 
     name = "compile"
     haltOnFailure = 1
+    flunkOnFailure = 1
     description = ["compiling"]
     descriptionDone = ["compile"]
     command = ["make", "all"]
 
     OFFprogressMetrics = ('output',)
     # things to track: number of files compiled, number of directories
     # traversed (assuming 'make' is being used)
 
-    def createSummary(self, cmd):
+    def createSummary(self, log):
         # TODO: grep for the characteristic GCC error lines and
         # assemble them into a pair of buffers
-        WarningCountingShellCommand.createSummary(self, cmd)
-        pass
+        WarningCountingShellCommand.createSummary(self, log)
 
 class Test(WarningCountingShellCommand):
 
     name = "test"
     warnOnFailure = 1
     description = ["testing"]
     descriptionDone = ["test"]
     command = ["make", "test"]
@@ -396,52 +407,81 @@ class Test(WarningCountingShellCommand):
                 if total:
                     description.append('%d tests' % total)
                 if passed:
                     description.append('%d passed' % passed)
                 if warnings:
                     description.append('%d warnings' % warnings)
                 if failed:
                     description.append('%d failed' % failed)
-            else:
-                description.append("no test results")
         return description
 
 class PerlModuleTest(Test):
     command=["prove", "--lib", "lib", "-r", "t"]
     total = 0
 
     def evaluateCommand(self, cmd):
-        lines = self.getLog('stdio').readlines()
+        # Get stdio, stripping pesky newlines etc.
+        lines = map(
+            lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''),
+            self.getLog('stdio').readlines()
+            )
 
-        re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),")
+        total = 0
+        passed = 0
+        failed = 0
+        rc = cmd.rc
 
-        mos = map(lambda line: re_test_result.search(line), lines)
-        test_result_lines = [mo.groups() for mo in mos if mo]
+        # New version of Test::Harness?
+        try:
+            test_summary_report_index = lines.index("Test Summary Report")
 
-        if not test_result_lines:
-            return cmd.rc
+            del lines[0:test_summary_report_index + 2]
 
-        test_result_line = test_result_lines[0]
+            re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)")
+
+            mos = map(lambda line: re_test_result.search(line), lines)
+            test_result_lines = [mo.groups() for mo in mos if mo]
 
-        success = test_result_line[0]
-
-        if success:
-            failed = 0
+            for line in test_result_lines:
+                if line[0] == 'PASS':
+                    rc = SUCCESS
+                elif line[0] == 'FAIL':
+                    rc = FAILURE
+                elif line[1]:
+                    failed += int(line[1])
+                elif line[2]:
+                    total = int(line[2])
 
-            test_totals_line = test_result_lines[1]
-            total_str = test_totals_line[3]
+        except ValueError: # Nope, it's the old version
+            re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),")
+
+            mos = map(lambda line: re_test_result.search(line), lines)
+            test_result_lines = [mo.groups() for mo in mos if mo]
 
-            rc = SUCCESS
-        else:
-            failed_str = test_result_line[1]
-            failed = int(failed_str)
+            if test_result_lines:
+                test_result_line = test_result_lines[0]
+
+                success = test_result_line[0]
+
+                if success:
+                    failed = 0
 
-            total_str = test_result_line[2]
-
-            rc = FAILURE
+                    test_totals_line = test_result_lines[1]
+                    total_str = test_totals_line[3]
+                    
+                    rc = SUCCESS
+                else:
+                    failed_str = test_result_line[1]
+                    failed = int(failed_str)
 
-        total = int(total_str)
-        passed = total - failed
+                    total_str = test_result_line[2]
+
+                    rc = FAILURE
 
-        self.setTestResults(total=total, failed=failed, passed=passed)
+                total = int(total_str)
+
+        if total:
+            passed = total - failed
+
+            self.setTestResults(total=total, failed=failed, passed=passed)
 
         return rc
--- a/buildbot/steps/source.py
+++ b/buildbot/steps/source.py
@@ -13,16 +13,17 @@ class Source(LoggingBuildStep):
     Each version control system has a specialized subclass, and is expected
     to override __init__ and implement computeSourceRevision() and
     startVC(). The class as a whole builds up the self.args dictionary, then
     starts a LoggedRemoteCommand with those arguments.
     """
 
     # if the checkout fails, there's no point in doing anything else
     haltOnFailure = True
+    flunkOnFailure = True
     notReally = False
 
     branch = None # the default branch, should be set in __init__
 
     def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
                  timeout=20*60, retry=None, **kwargs):
         """
         @type  workdir: string
@@ -154,17 +155,16 @@ class Source(LoggingBuildStep):
         systems that do not (CVS), it needs to create a timestamp based upon
         the latest Change, the Build's treeStableTimer, and an optional
         self.checkoutDelay value."""
         return None
 
     def start(self):
         if self.notReally:
             log.msg("faking %s checkout/update" % self.name)
-            self.step_status.setColor("green")
             self.step_status.setText(["fake", self.name, "successful"])
             self.addCompleteLog("log",
                                 "Faked %s checkout/update 'successful'\n" \
                                 % self.name)
             return SKIPPED
 
         # what source stamp would this build like to use?
         s = self.build.getSourceStamp()
@@ -179,20 +179,20 @@ class Source(LoggingBuildStep):
         # 'patch' is None or a tuple of (patchlevel, diff)
         patch = s.patch
         if patch:
             self.addCompleteLog("patch", patch[1])
 
         self.startVC(branch, revision, patch)
 
     def commandComplete(self, cmd):
-        got_revision = None
         if cmd.updates.has_key("got_revision"):
-            got_revision = str(cmd.updates["got_revision"][-1])
-        self.setProperty("got_revision", got_revision, "Source")
+            got_revision = cmd.updates["got_revision"][-1]
+            if got_revision is not None:
+                self.setProperty("got_revision", str(got_revision), "Source")
 
 
 
 class CVS(Source):
     """I do CVS checkout/update operations.
 
     Note: if you are doing anonymous/pserver CVS operations, you will need
     to manually do a 'cvs login' on each buildslave before the slave has any
@@ -346,17 +346,17 @@ class CVS(Source):
 
 
 class SVN(Source):
     """I perform Subversion checkout/update operations."""
 
     name = 'svn'
 
     def __init__(self, svnurl=None, baseURL=None, defaultBranch=None,
-                 directory=None, **kwargs):
+                 directory=None, username=None, password=None, **kwargs):
         """
         @type  svnurl: string
         @param svnurl: the URL which points to the Subversion server,
                        combining the access method (HTTP, ssh, local file),
                        the repository host/port, the repository path, the
                        sub-tree within the repository, and the branch to
                        check out. Using C{svnurl} does not enable builds of
                        alternate branches: use C{baseURL} to enable this.
@@ -367,32 +367,39 @@ class SVN(Source):
                         probably end in a slash. Use exactly one of
                         C{svnurl} and C{baseURL}.
 
         @param defaultBranch: if branches are enabled, this is the branch
                               to use if the Build does not specify one
                               explicitly. It will simply be appended
                               to C{baseURL} and the result handed to
                               the SVN command.
+
+        @param username: username to pass to svn's --username
+        @param password: username to pass to svn's --password
         """
 
         if not kwargs.has_key('workdir') and directory is not None:
             # deal with old configs
             warn("Please use workdir=, not directory=", DeprecationWarning)
             kwargs['workdir'] = directory
 
         self.svnurl = svnurl
         self.baseURL = baseURL
         self.branch = defaultBranch
+        self.username = username
+        self.password = password
 
         Source.__init__(self, **kwargs)
         self.addFactoryArguments(svnurl=svnurl,
                                  baseURL=baseURL,
                                  defaultBranch=defaultBranch,
                                  directory=directory,
+                                 username=username,
+                                 password=password,
                                  )
 
         if not svnurl and not baseURL:
             raise ValueError("you must use exactly one of svnurl and baseURL")
 
 
     def computeSourceRevision(self, changes):
         if not changes or None in [c.revision for c in changes]:
@@ -454,16 +461,26 @@ class SVN(Source):
         if self.svnurl:
             assert not branch # we need baseURL= to use branches
             self.args['svnurl'] = self.svnurl
         else:
             self.args['svnurl'] = self.baseURL + branch
         self.args['revision'] = revision
         self.args['patch'] = patch
 
+        if self.username is not None or self.password is not None:
+            if self.slaveVersionIsOlderThan("svn", "2.8"):
+                m = ("This buildslave (%s) does not support svn usernames "
+                     "and passwords.  "
+                     "Refusing to build. Please upgrade the buildslave to "
+                     "buildbot-0.7.10 or newer." % (self.build.slavename,))
+                raise BuildSlaveTooOldError(m)
+            if self.username is not None: self.args['username'] = self.username
+            if self.password is not None: self.args['password'] = self.password
+
         revstuff = []
         if branch is not None and branch != self.branch:
             revstuff.append("[branch]")
         if revision is not None:
             revstuff.append("r%s" % revision)
         if patch is not None:
             revstuff.append("[patch]")
         self.description.extend(revstuff)
@@ -841,57 +858,71 @@ class Bzr(Source):
 
 
 class Mercurial(Source):
     """Check out a source tree from a mercurial repository 'repourl'."""
 
     name = "hg"
 
     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
-                 **kwargs):
+                 branchType='dirname', **kwargs):
         """
         @type  repourl: string
         @param repourl: the URL which points at the Mercurial repository.
-                        This is used as the default branch. Using C{repourl}
-                        does not enable builds of alternate branches: use
-                        C{baseURL} to enable this. Use either C{repourl} or
-                        C{baseURL}, not both.
+                        This uses the 'default' branch unless defaultBranch is
+                        specified below and the C{branchType} is set to
+                        'inrepo'.  It is an error to specify a branch without
+                        setting the C{branchType} to 'inrepo'.
 
-        @param baseURL: if branches are enabled, this is the base URL to
-                        which a branch name will be appended. It should
-                        probably end in a slash. Use exactly one of
-                        C{repourl} and C{baseURL}.
+        @param baseURL: if 'dirname' branches are enabled, this is the base URL
+                        to which a branch name will be appended. It should
+                        probably end in a slash.  Use exactly one of C{repourl}
+                        and C{baseURL}.
 
         @param defaultBranch: if branches are enabled, this is the branch
                               to use if the Build does not specify one
-                              explicitly. It will simply be appended to
-                              C{baseURL} and the result handed to the
-                              'hg clone' command.
+                              explicitly.
+                              For 'dirname' branches, It will simply be
+                              appended to C{baseURL} and the result handed to
+                              the 'hg update' command.
+                              For 'inrepo' branches, this specifies the named
+                              revision to which the tree will update after a
+                              clone.
+
+        @param branchType: either 'dirname' or 'inrepo' depending on whether
+                           the branch name should be appended to the C{baseURL}
+                           or the branch is a mercurial named branch and can be
+                           found within the C{repourl}
         """
         self.repourl = repourl
         self.baseURL = baseURL
         self.branch = defaultBranch
+        self.branchType = branchType
         Source.__init__(self, **kwargs)
         self.addFactoryArguments(repourl=repourl,
                                  baseURL=baseURL,
                                  defaultBranch=defaultBranch,
+                                 branchType=branchType,
                                  )
         if (not repourl and not baseURL) or (repourl and baseURL):
             raise ValueError("you must provide exactly one of repourl and"
                              " baseURL")
 
     def startVC(self, branch, revision, patch):
         slavever = self.slaveVersion("hg")
         if not slavever:
             raise BuildSlaveTooOldError("slave is too old, does not know "
                                         "about hg")
 
         if self.repourl:
-            assert not branch # we need baseURL= to use branches
+            # we need baseURL= to use dirname branches
+            assert self.branchType == 'inrepo' or not branch
             self.args['repourl'] = self.repourl
+            if branch:
+                self.args['branch'] = branch
         else:
             self.args['repourl'] = self.baseURL + branch
         self.args['revision'] = revision
         self.args['patch'] = patch
 
         revstuff = []
         if branch is not None and branch != self.branch:
             revstuff.append("[branch]")
--- a/buildbot/steps/transfer.py
+++ b/buildbot/steps/transfer.py
@@ -54,32 +54,119 @@ class _FileWriter(pb.Referenceable):
         # unclean shutdown, the file is probably truncated, so delete it
         # altogether rather than deliver a corrupted file
         fp = getattr(self, "fp", None)
         if fp:
             fp.close()
             os.unlink(self.destfile)
 
 
+class _DirectoryWriter(pb.Referenceable):
+    """
+    Helper class that acts as a directory-object with write access
+    """
+
+    def __init__(self, destroot, maxsize, mode):
+        self.destroot = destroot
+	# Create missing directories.
+        self.destroot = os.path.abspath(self.destroot)
+        if not os.path.exists(self.destroot):
+            os.makedirs(self.destroot)
+	
+	self.fp = None
+	self.mode = mode
+	self.maxsize = maxsize
+
+    def remote_createdir(self, dirname):
+	# This function is needed to transfer empty directories.
+	dirname = os.path.join(self.destroot, dirname)
+	dirname = os.path.abspath(dirname)
+	if not os.path.exists(dirname):
+            os.makedirs(dirname)
+
+    def remote_open(self, destfile):
+	# Create missing directories.
+	destfile = os.path.join(self.destroot, destfile)
+        destfile = os.path.abspath(destfile)
+        dirname = os.path.dirname(destfile)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+
+        self.fp = open(destfile, "wb")
+        if self.mode is not None:
+            os.chmod(destfile, self.mode)
+	self.remaining = self.maxsize
+
+    def remote_write(self, data):
+	"""
+	Called from remote slave to write L{data} to L{fp} within boundaries
+	of L{maxsize}
+
+	@type  data: C{string}
+	@param data: String of data to write
+	"""
+        if self.remaining is not None:
+            if len(data) > self.remaining:
+                data = data[:self.remaining]
+            self.fp.write(data)
+            self.remaining = self.remaining - len(data)
+        else:
+            self.fp.write(data)
+
+    def remote_close(self):
+        """
+        Called by remote slave to state that no more data will be transfered
+        """
+	if self.fp:
+	    self.fp.close()
+    	    self.fp = None
+
+    def __del__(self):
+        # unclean shutdown, the file is probably truncated, so delete it
+        # altogether rather than deliver a corrupted file
+        fp = getattr(self, "fp", None)
+        if fp:
+            fp.close()
+
+
 class StatusRemoteCommand(RemoteCommand):
     def __init__(self, remote_command, args):
         RemoteCommand.__init__(self, remote_command, args)
 
         self.rc = None
         self.stderr = ''
 
     def remoteUpdate(self, update):
         #log.msg('StatusRemoteCommand: update=%r' % update)
         if 'rc' in update:
             self.rc = update['rc']
         if 'stderr' in update:
             self.stderr = self.stderr + update['stderr'] + '\n'
 
+class _TransferBuildStep(BuildStep):
+    """
+    Base class for FileUpload and FileDownload to factor out common
+    functionality.
+    """
+    DEFAULT_WORKDIR = "build"           # is this redundant?
 
-class FileUpload(BuildStep):
+    def setDefaultWorkdir(self, workdir):
+        if self.workdir is None:
+            self.workdir = workdir
+
+    def _getWorkdir(self):
+        properties = self.build.getProperties()
+        if self.workdir is None:
+            workdir = self.DEFAULT_WORKDIR
+        else:
+            workdir = self.workdir
+        return properties.render(workdir)
+
+
+class FileUpload(_TransferBuildStep):
     """
     Build step to transfer a file from the slave to the master.
 
     arguments:
 
     - ['slavesrc']   filename of source file at slave, relative to workdir
     - ['masterdest'] filename of destination file at master
     - ['workdir']    string with slave working directory relative to builder
@@ -90,17 +177,17 @@ class FileUpload(BuildStep):
                      The default (=None) is to leave it up to the umask of
                      the buildmaster process.
 
     """
 
     name = 'upload'
 
     def __init__(self, slavesrc, masterdest,
-                 workdir="build", maxsize=None, blocksize=16*1024, mode=None,
+                 workdir=None, maxsize=None, blocksize=16*1024, mode=None,
                  **buildstep_kwargs):
         BuildStep.__init__(self, **buildstep_kwargs)
         self.addFactoryArguments(slavesrc=slavesrc,
                                  masterdest=masterdest,
                                  workdir=workdir,
                                  maxsize=maxsize,
                                  blocksize=blocksize,
                                  mode=mode,
@@ -127,46 +214,127 @@ class FileUpload(BuildStep):
         # we rely upon the fact that the buildmaster runs chdir'ed into its
         # basedir to make sure that relative paths in masterdest are expanded
         # properly. TODO: maybe pass the master's basedir all the way down
         # into the BuildStep so we can do this better.
         masterdest = os.path.expanduser(masterdest)
         log.msg("FileUpload started, from slave %r to master %r"
                 % (source, masterdest))
 
-        self.step_status.setColor('yellow')
         self.step_status.setText(['uploading', os.path.basename(source)])
 
         # we use maxsize to limit the amount of data on both sides
         fileWriter = _FileWriter(masterdest, self.maxsize, self.mode)
 
         # default arguments
         args = {
             'slavesrc': source,
-            'workdir': self.workdir,
+            'workdir': self._getWorkdir(),
             'writer': fileWriter,
             'maxsize': self.maxsize,
             'blocksize': self.blocksize,
             }
 
         self.cmd = StatusRemoteCommand('uploadFile', args)
         d = self.runCommand(self.cmd)
         d.addCallback(self.finished).addErrback(self.failed)
 
     def finished(self, result):
         if self.cmd.stderr != '':
             self.addCompleteLog('stderr', self.cmd.stderr)
 
         if self.cmd.rc is None or self.cmd.rc == 0:
-            self.step_status.setColor('green')
             return BuildStep.finished(self, SUCCESS)
-        self.step_status.setColor('red')
         return BuildStep.finished(self, FAILURE)
 
 
+class DirectoryUpload(BuildStep):
+    """
+    Build step to transfer a directory from the slave to the master.
+
+    arguments:
+
+    - ['slavesrc']   name of source directory at slave, relative to workdir
+    - ['masterdest'] name of destination directory at master
+    - ['workdir']    string with slave working directory relative to builder
+                     base dir, default 'build'
+    - ['maxsize']    maximum size of each file, default None (=unlimited)
+    - ['blocksize']  maximum size of each block being transfered
+    - ['mode']       file access mode for the resulting master-side file.
+                     The default (=None) is to leave it up to the umask of
+                     the buildmaster process.
+
+    """
+
+    name = 'upload'
+
+    def __init__(self, slavesrc, masterdest,
+                 workdir="build", maxsize=None, blocksize=16*1024, mode=None,
+                 **buildstep_kwargs):
+        BuildStep.__init__(self, **buildstep_kwargs)
+        self.addFactoryArguments(slavesrc=slavesrc,
+                                 masterdest=masterdest,
+                                 workdir=workdir,
+                                 maxsize=maxsize,
+                                 blocksize=blocksize,
+                                 mode=mode,
+                                 )
+
+        self.slavesrc = slavesrc
+        self.masterdest = masterdest
+        self.workdir = workdir
+        self.maxsize = maxsize
+        self.blocksize = blocksize
+        assert isinstance(mode, (int, type(None)))
+        self.mode = mode
+
+    def start(self):
+        version = self.slaveVersion("uploadDirectory")
+        properties = self.build.getProperties()
+
+        if not version:
+            m = "slave is too old, does not know about uploadDirectory"
+            raise BuildSlaveTooOldError(m)
+
+        source = properties.render(self.slavesrc)
+        masterdest = properties.render(self.masterdest)
+        # we rely upon the fact that the buildmaster runs chdir'ed into its
+        # basedir to make sure that relative paths in masterdest are expanded
+        # properly. TODO: maybe pass the master's basedir all the way down
+        # into the BuildStep so we can do this better.
+        masterdest = os.path.expanduser(masterdest)
+        log.msg("DirectoryUpload started, from slave %r to master %r"
+                % (source, masterdest))
+
+        self.step_status.setText(['uploading', os.path.basename(source)])
+	
+        # we use maxsize to limit the amount of data on both sides
+        dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.mode)
+
+        # default arguments
+        args = {
+            'slavesrc': source,
+            'workdir': self.workdir,
+            'writer': dirWriter,
+            'maxsize': self.maxsize,
+            'blocksize': self.blocksize,
+            }
+
+        self.cmd = StatusRemoteCommand('uploadDirectory', args)
+        d = self.runCommand(self.cmd)
+        d.addCallback(self.finished).addErrback(self.failed)
+
+    def finished(self, result):
+        if self.cmd.stderr != '':
+            self.addCompleteLog('stderr', self.cmd.stderr)
+
+        if self.cmd.rc is None or self.cmd.rc == 0:
+            return BuildStep.finished(self, SUCCESS)
+        return BuildStep.finished(self, FAILURE)
+
 
 
 
 class _FileReader(pb.Referenceable):
     """
     Helper class that acts as a file-object with read access
     """
 
@@ -193,17 +361,17 @@ class _FileReader(pb.Referenceable):
         """
         Called by remote slave to state that no more data will be transfered
         """
         if self.fp is not None:
             self.fp.close()
             self.fp = None
 
 
-class FileDownload(BuildStep):
+class FileDownload(_TransferBuildStep):
     """
     Download the first 'maxsize' bytes of a file, from the buildmaster to the
     buildslave. Set the mode of the file
 
     Arguments::
 
      ['mastersrc'] filename of source file at master
      ['slavedest'] filename of destination file at slave
@@ -215,21 +383,20 @@ class FileDownload(BuildStep):
                    buildslave-side file. This is traditionally an octal
                    integer, like 0644 to be world-readable (but not
                    world-writable), or 0600 to only be readable by
                    the buildslave account, or 0755 to be world-executable.
                    The default (=None) is to leave it up to the umask of
                    the buildslave process.
 
     """
-
     name = 'download'
 
     def __init__(self, mastersrc, slavedest,
-                 workdir="build", maxsize=None, blocksize=16*1024, mode=None,
+                 workdir=None, maxsize=None, blocksize=16*1024, mode=None,
                  **buildstep_kwargs):
         BuildStep.__init__(self, **buildstep_kwargs)
         self.addFactoryArguments(mastersrc=mastersrc,
                                  slavedest=slavedest,
                                  workdir=workdir,
                                  maxsize=maxsize,
                                  blocksize=blocksize,
                                  mode=mode,
@@ -253,17 +420,16 @@ class FileDownload(BuildStep):
 
         # we are currently in the buildmaster's basedir, so any non-absolute
         # paths will be interpreted relative to that
         source = os.path.expanduser(properties.render(self.mastersrc))
         slavedest = properties.render(self.slavedest)
         log.msg("FileDownload started, from master %r to slave %r" %
                 (source, slavedest))