--- a/.coveragerc
+++ b/.coveragerc
@@ -12,16 +12,19 @@ exclude_lines =
raise AssertionError
raise NotImplementedError
# Don't complain if non-runnable code isn't run:
if 0:
if __name__ == .__main__.:
if runtime.platformType == 'win32'
+ # 'pass' generally means 'this won't be called'
+ ^ *pass *$
+
include =
master/*
slave/*
omit =
# omit all of our tests
*/test/*
# templates cause coverage errors
--- a/.gitignore
+++ b/.gitignore
@@ -31,8 +31,9 @@ master/docs/buildbot
master/docs/version.texinfo
master/docs/docs.tgz
master/docs/latest
twisted/plugins/dropin.cache
.coverage
coverage-html
apidocs/reference
apidocs/reference.tgz
+common/googlecode_upload.py
--- a/Makefile
+++ b/Makefile
@@ -6,8 +6,11 @@ docs:
$(MAKE) -C master/docs
apidocs:
$(MAKE) -C apidocs
pylint:
cd master; $(MAKE) pylint
cd slave; $(MAKE) pylint
+
+pyflakes:
+ pyflakes master/buildbot slave/buildslave
new file mode 100644
--- /dev/null
+++ b/common/gcode-upload.sh
@@ -0,0 +1,63 @@
+#! /bin/sh
+
+usage='USAGE: gcode-upload.sh VERSION USERNAME GCODE-PASSWORD
+
+The gcode password is available from http://code.google.com/hosting/settings;
+it is *not* your google account password.
+'
+
+if test $# != 3; then
+ echo "$usage"
+ exit 1
+fi
+
+if test ! -f common/googlecode_upload.py; then
+ echo "download googlecode_upload.py from"
+ echo " http://support.googlecode.com/svn/trunk/scripts/googlecode_upload.py"
+ echo "and place it in common/"
+ exit 1
+fi
+
+VERSION="$1"
+USERNAME="$2"
+PASSWORD="$3"
+
+findfile() {
+ local file="$1"
+ test -f "dist/$file" && echo "dist/$file"
+ test -f "master/dist/$file" && echo "master/dist/$file"
+ test -f "slave/dist/$file" && echo "slave/dist/$file"
+ test -f "$file" && echo "$file"
+}
+
+findlabels() {
+ local file="$1"
+ local labels=Featured
+ if test "`echo $file | sed 's/.*\.asc/Signature/'`" = "Signature"; then
+ labels="$labels,Signature"
+ fi
+ if test "`echo $file | sed 's/.*\.tar.gz.*/Tar/'`" = "Tar"; then
+ labels="$labels,OpSys-POSIX"
+ else
+ labels="$labels,OpSys-Win"
+ fi
+ echo $labels
+}
+
+i=0
+for file in {buildbot,buildbot-slave}-$VERSION.{tar.gz,zip}{,.asc}; do
+ if test $i = 0; then
+ i=1
+ continue
+ fi
+ labels=`findlabels "$file"`
+ file=`findfile "$file"`
+ echo "Uploading $file with labels $labels"
+ python common/googlecode_upload.py \
+ -w $PASSWORD \
+ -u $USERNAME \
+ -p buildbot \
+ -s "$file" \
+ --labels=Featured \
+ "$file"
+done
--- a/master/COPYING
+++ b/master/COPYING
@@ -273,67 +273,8 @@ REDISTRIBUTE THE PROGRAM AS PERMITTED AB
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
-
- How to Apply These Terms to Your New Programs
-
- If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
- To do so, attach the following notices to the program. It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
- <one line to give the program's name and a brief idea of what it does.>
- Copyright (C) <year> <name of author>
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
- Gnomovision version 69, Copyright (C) year name of author
- Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
- This is free software, and you are welcome to redistribute it
- under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License. Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary. Here is a sample; alter the names:
-
- Yoyodyne, Inc., hereby disclaims all copyright interest in the program
- `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
- <signature of Ty Coon>, 1 April 1989
- Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs. If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library. If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.
--- a/master/MANIFEST.in
+++ b/master/MANIFEST.in
@@ -1,17 +1,17 @@
include MANIFEST.in README NEWS CREDITS COPYING UPGRADING
include docs/examples/*.cfg
include docs/*.texinfo
include docs/*.png docs/images/*.png docs/images/*.svg docs/images/*.txt docs/images/*.ai
include docs/images/icon.blend docs/images/Makefile
-include docs/epyrun docs/gen-reference docs/Makefile
+include docs/Makefile
include docs/version.py
-include docs/buildbot.1 docs/buildslave.1
+include docs/buildbot.1
include buildbot/db/schema/tables.sql
include buildbot/scripts/sample.cfg
include buildbot/status/web/files/*
include buildbot/status/web/templates/*.html buildbot/status/web/templates/*.xml
include buildbot/clients/debug.glade
include buildbot/buildbot.png
--- a/master/Makefile
+++ b/master/Makefile
@@ -1,3 +1,6 @@
# developer utilities
pylint:
pylint --rcfile=../common/pylintrc buildbot
+
+pyflakes:
+ pyflakes buildbot
--- a/master/NEWS
+++ b/master/NEWS
@@ -1,13 +1,67 @@
Major User visible changes in Buildbot. -*- outline -*-
see the git log for a detailed list of changes:
http://github.com/buildbot/buildbot/commits/master
-* Buildbot 0.8.2
+* Buildbot 0.8.3p1
+
+** Critical Fixes for GitPoller
+
+*** correctly initialize a new repository with 'git init' and 'git fetch',
+albiet using blocking calls (fixes #1742)
+
+*** correctly synchronize processing of each change in a large batch of changes
+(fixes #1745)
+
+* Buildbot 0.8.3 (December 19, 2010)
+
+** PBChangeSource now supports authentication
+
+PBChangeSource now supports the `user` and `passwd` arguments. Users with a
+publicly exposed PB port should use these parameters to limit sendchange
+access.
+
+Previous versions of Buildbot should never be configured with a PBChangeSource
+and a publicly accessible slave port, as that arrangement allows anyone to
+connect and inject a change into the Buildmaster without any authentication at
+all, aside from the hard-coded 'change'/'changepw' credentials. In many cases,
+this can lead to arbitrary code injection on slaves.
+
+** Experiemental Gerrit and Repo support
+
+A new ChangeSource (GerritChangeSource), status listener (GerritStatusPush),
+and source step (Repo) are available in this version. These are not fully
+documented and still have a number of known bugs outstanding (see
+http://buildbot.net/trac/wiki/RepoProject), and as such are considered
+experimental in this release.
+
+** WithProperties now supports lambda substitutions
+
+WithProperties now has the option to pass callable functions as keyword
+arguments to substitute in the results of more complex Python code at
+evaluation-time.
+
+** New 'SetPropertiesFromEnv' step
+
+This step uses the slave environment to set build properties.
+
+** Deprecations and Removals
+
+*** The console view previously had an undocumented feature that would strip
+leading digits off the category name. This was undocumented and apparently
+non-functional, and has been removed. (#1059)
+
+*** contrib/hg_buildbot.py was removed in favor of buildbot.changes.hgbuildbot.
+
+*** The misnamed sendchange option 'username' has been renamed to 'who'; the old
+option continues to work, but is deprecated and will be removed. (#1711)
+
+
+* Buildbot 0.8.2 (October 29, 2010)
** Upgrading
Upgrading to from the previous version will require an 'upgrade-master' run.
However, the schema changes are backward-compatible, so if a downgrade is
required, it will not be difficult.
** New Requirements
@@ -79,16 +133,17 @@ See docs for more info.
Bonsai BonsaiMaildirSource
*** statusgui is deprecated in this version and will be removed in the next
release. Please file a bug at http://buildbot.net if you wish to reverse this
decision.
*** The Twisted-only steps BuildDebs and ProcessDocs have been removed.
+
* Release 0.8.1 (June 16, 2010)
** Slave Split into separate component
Installing 'buildbot' will no longer allow you to run a slave - for that,
you'll now need the 'buildslave' component, which is available by easy_install.
This is merely a packaging change - the buildslave and buildbot components are
completely inter-compatible, just as they always have been.
@@ -134,16 +189,17 @@ them.
*** Support for starting buildmaster from Makefiles to be removed in 0.8.2
In a little-used feature, 'buildbot start' would run 'make start' if a
Makefile.buildbot existed in the master directory. This functionality will be
removed in Buildbot-0.8.2, and the create-master command will no longer create
a Makefile.sample. Of course, Buildbot still supports build processes on the
slave using make!
+
* Release 0.8.0 (May 25, 2010)
** (NOTE!) Scheduler requires keyword arguments
If you are creating your Scheduler like this:
Scheduler("mysched", "mybranch", 0, ["foo", "bar"])
@@ -223,16 +279,17 @@ has seen a lot of house-cleaning, and ev
*** Removed buildbot.status.html.Waterfall (deprecated in 0.7.6)
Note that this does not remove the waterfall -- just an old version of it which
did not include the rest of the WebStatus pages.
*** BuildmasterConfig no longer accepts 'bots' and 'sources' as keys
(deprecated in 0.7.6). Use 'slaves' and 'change_source' instead.
+
* Release 0.7.12 (January 21, 2010)
** New 'console' display
This is a new web status view combining the best of the (t)grid and waterfall
views.
** New 'extended' stylesheet
@@ -294,21 +351,23 @@ information about test runs and test fai
** Python API Docs
The docstrings for buildbot are now available in a web-friendly format:
http://buildbot.net/buildbot/docs/latest/reference
** Many, many bugfixes
+
* Release 0.7.11p (July 16, 2009)
Fixes a few test failures in 0.7.11, and gives a default value for branchType
if it is not specified by the master.
+
* Release 0.7.11 (July 5, 2009)
Developers too numerous to mention contributed to this release. Buildbot has
truly become a community-maintained application. Much hard work is not
mentioned here, so please consult the git logs for the detailed changes in this
release.
** Better Memory Performance, Disk Cleanup
@@ -359,16 +418,17 @@ Bugfixes and fairer scheduling
Similar to the grid view, but with the axes reversed and showing different
info. Located at /tgrid.
** Trigger steps improvements
Trigger now supports copy_properties, to send selected properties to the
triggered build.
+
* Release 0.7.10 (25 Feb 2009)
This release is mainly a collection of user-submitted patches since
the last release.
** New Features
*** Environment variables in a builder (#100)
@@ -534,16 +594,17 @@ can use f.addSteps(steplist) to add them
By default, Buildbot now rotates and cleans up the (potentially voluminous)
twistd.log files.
*** Prioritize build requests based on the time they wre submitted (#334)
Balancing of load is a bit more fair, although not true load balancing.
+
* Release 0.7.9 (15 Sep 2008)
** New Features
*** Configurable public_html directory (#162)
The public_html/ directory, which provides static content for the WebStatus()
HTTP server, is now configurable. The default location is still the
--- a/master/buildbot/__init__.py
+++ b/master/buildbot/__init__.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# strategy:
#
# if there is a VERSION file, use its contents. otherwise, call git to
# get a version string. if that also fails, use 'latest'.
#
import os
version = "latest"
--- a/master/buildbot/buildrequest.py
+++ b/master/buildbot/buildrequest.py
@@ -1,44 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
from buildbot import interfaces
from buildbot.process.properties import Properties
class BuildRequest:
"""I represent a request to a specific Builder to run a single build.
I am generated by db.getBuildRequestWithNumber, and am used to tell the
--- a/master/buildbot/buildslave.py
+++ b/master/buildbot/buildslave.py
@@ -1,27 +1,39 @@
-# Portions copyright Canonical Ltd. 2009
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Portions Copyright Buildbot Team Members
+# Portions Copyright Canonical Ltd. 2009
import time
from email.Message import Message
from email.Utils import formatdate
from zope.interface import implements
from twisted.python import log
from twisted.internet import defer, reactor
from twisted.application import service
from twisted.spread import pb
from buildbot.status.builder import SlaveStatus
from buildbot.status.mail import MailNotifier
from buildbot.interfaces import IBuildSlave, ILatentBuildSlave
from buildbot.process.properties import Properties
from buildbot.locks import LockAccess
-import sys
-
class AbstractBuildSlave(pb.Avatar, service.MultiService):
"""This is the master-side representative for a remote buildbot slave.
There is exactly one for each slave described in the config file (the
c['slaves'] list). When buildbots connect in (.attach), they get a
reference to this instance. The BotMaster object is stashed as the
.botmaster attribute. The BotMaster is also our '.parent' Service.
I represent a build slave -- a remote machine capable of
@@ -246,16 +258,19 @@ class AbstractBuildSlave(pb.Avatar, serv
def _get_info(res):
d1 = bot.callRemote("getSlaveInfo")
def _got_info(info):
log.msg("Got slaveinfo from '%s'" % self.slavename)
# TODO: info{} might have other keys
state["admin"] = info.get("admin")
state["host"] = info.get("host")
state["access_uri"] = info.get("access_uri", None)
+ state["slave_environ"] = info.get("environ", {})
+ state["slave_basedir"] = info.get("basedir", None)
+ state["slave_system"] = info.get("system", None)
def _info_unavailable(why):
# maybe an old slave, doesn't implement remote_getSlaveInfo
log.msg("BuildSlave.info_unavailable")
log.err(why)
d1.addCallbacks(_got_info, _info_unavailable)
return d1
d.addCallback(_get_info)
@@ -286,16 +301,19 @@ class AbstractBuildSlave(pb.Avatar, serv
def _accept_slave(res):
self.slave_status.setAdmin(state.get("admin"))
self.slave_status.setHost(state.get("host"))
self.slave_status.setAccessURI(state.get("access_uri"))
self.slave_status.setVersion(state.get("version"))
self.slave_status.setConnected(True)
self.slave_commands = state.get("slave_commands")
+ self.slave_environ = state.get("slave_environ")
+ self.slave_basedir = state.get("slave_basedir")
+ self.slave_system = state.get("slave_system")
self.slave = bot
log.msg("bot attached")
self.messageReceivedFromSlave()
self.stopMissingTimer()
self.botmaster.parent.status.slaveConnected(self.slavename)
return self.updateSlave()
d.addCallback(_accept_slave)
@@ -379,16 +397,20 @@ class AbstractBuildSlave(pb.Avatar, serv
our_builders = self.botmaster.getBuildersForSlave(self.slavename)
blist = [(b.name, b.slavebuilddir) for b in our_builders]
d = self.slave.callRemote("setBuilderList", blist)
return d
def perspective_keepalive(self):
pass
+ def perspective_shutdown(self):
+ log.msg("slave %s wants to shut down" % self.slavename)
+ self.slave_status.setGraceful(True)
+
def addSlaveBuilder(self, sb):
self.slavebuilders[sb.builder_name] = sb
def removeSlaveBuilder(self, sb):
try:
del self.slavebuilders[sb.builder_name]
except KeyError:
pass
@@ -437,51 +459,88 @@ class AbstractBuildSlave(pb.Avatar, serv
recipients = self.notify_on_missing
m['To'] = ", ".join(recipients)
d = st.sendMessage(m, recipients)
# return the Deferred for testing purposes
return d
def _gracefulChanged(self, graceful):
"""This is called when our graceful shutdown setting changes"""
- if graceful:
- active_builders = [sb for sb in self.slavebuilders.values()
- if sb.isBusy()]
- if len(active_builders) == 0:
- # Shut down!
- self.shutdown()
+ self.maybeShutdown()
+ @defer.deferredGenerator
def shutdown(self):
"""Shutdown the slave"""
- # Look for a builder with a remote reference to the client side
- # slave. If we can find one, then call "shutdown" on the remote
- # builder, which will cause the slave buildbot process to exit.
- d = None
- for b in self.slavebuilders.values():
- if b.remote:
- d = b.remote.callRemote("shutdown")
- break
+ if not self.slave:
+ log.msg("no remote; slave is already shut down")
+ return
+
+ # First, try the "new" way - calling our own remote's shutdown
+ # method. The method was only added in 0.8.3, so ignore NoSuchMethod
+ # failures.
+ def new_way():
+ d = self.slave.callRemote('shutdown')
+ d.addCallback(lambda _ : True) # successful shutdown request
+ def check_nsm(f):
+ f.trap(pb.NoSuchMethod)
+ return False # fall through to the old way
+ d.addErrback(check_nsm)
+ def check_connlost(f):
+ f.trap(pb.PBConnectionLost)
+ return True # the slave is gone, so call it finished
+ d.addErrback(check_connlost)
+ return d
+
+ wfd = defer.waitForDeferred(new_way())
+ yield wfd
+ if wfd.getResult():
+ return # done!
- if d:
- log.msg("Shutting down slave: %s" % self.slavename)
- # The remote shutdown call will not complete successfully since the
- # buildbot process exits almost immediately after getting the
- # shutdown request.
- # Here we look at the reason why the remote call failed, and if
- # it's because the connection was lost, that means the slave
- # shutdown as expected.
- def _errback(why):
- if why.check(pb.PBConnectionLost):
- log.msg("Lost connection to %s" % self.slavename)
- else:
- log.err("Unexpected error when trying to shutdown %s" % self.slavename)
- d.addErrback(_errback)
- return d
- log.err("Couldn't find remote builder to shut down slave")
- return defer.succeed(None)
+ # Now, the old way. Look for a builder with a remote reference to the
+ # client side slave. If we can find one, then call "shutdown" on the
+ # remote builder, which will cause the slave buildbot process to exit.
+ def old_way():
+ d = None
+ for b in self.slavebuilders.values():
+ if b.remote:
+ d = b.remote.callRemote("shutdown")
+ break
+
+ if d:
+ log.msg("Shutting down (old) slave: %s" % self.slavename)
+ # The remote shutdown call will not complete successfully since the
+ # buildbot process exits almost immediately after getting the
+ # shutdown request.
+ # Here we look at the reason why the remote call failed, and if
+ # it's because the connection was lost, that means the slave
+ # shutdown as expected.
+ def _errback(why):
+ if why.check(pb.PBConnectionLost):
+ log.msg("Lost connection to %s" % self.slavename)
+ else:
+ log.err("Unexpected error when trying to shutdown %s" % self.slavename)
+ d.addErrback(_errback)
+ return d
+ log.err("Couldn't find remote builder to shut down slave")
+ return defer.succeed(None)
+ #wfd = defer.waitForDeferred(old_way())
+ #yield wfd
+ #wfd.getResult()
+
+ def maybeShutdown(self):
+ """Shut down this slave if it has been asked to shut down gracefully,
+ and has no active builders."""
+ if not self.slave_status.getGraceful():
+ return
+ active_builders = [sb for sb in self.slavebuilders.values()
+ if sb.isBusy()]
+ if active_builders:
+ return
+ d = self.shutdown()
+ d.addErrback(log.err, 'error while shutting down slave')
class BuildSlave(AbstractBuildSlave):
def sendBuilderList(self):
d = AbstractBuildSlave.sendBuilderList(self)
def _sent(slist):
dl = []
for name, remote in slist.items():
@@ -503,22 +562,17 @@ class BuildSlave(AbstractBuildSlave):
AbstractBuildSlave.detached(self, mind)
self.botmaster.slaveLost(self)
self.startMissingTimer()
def buildFinished(self, sb):
"""This is called when a build on this slave is finished."""
# If we're gracefully shutting down, and we have no more active
# builders, then it's safe to disconnect
- if self.slave_status.getGraceful():
- active_builders = [sb for sb in self.slavebuilders.values()
- if sb.isBusy()]
- if len(active_builders) == 0:
- # Shut down!
- return self.shutdown()
+ self.maybeShutdown()
return defer.succeed(None)
class AbstractLatentBuildSlave(AbstractBuildSlave):
"""A build slave that will start up a slave instance when needed.
To use, subclass and implement start_instance and stop_instance.
See ec2buildslave.py for a concrete example. Also see the stub example in
@@ -691,17 +745,18 @@ class AbstractLatentBuildSlave(AbstractB
self.substantiation_deferred = None
if self.missing_timer:
self.missing_timer.cancel()
self.missing_timer = None
self.stop_instance()
return d
def disconnect(self):
- d = self._soft_disconnect()
+ # This returns a Deferred but we don't use it
+ self._soft_disconnect()
# this removes the slave from all builders. It won't come back
# without a restart (or maybe a sighup)
self.botmaster.slaveLost(self)
def stopService(self):
res = defer.maybeDeferred(AbstractBuildSlave.stopService, self)
if self.slave is not None:
d = self._soft_disconnect()
--- a/master/buildbot/changes/base.py
+++ b/master/buildbot/changes/base.py
@@ -1,10 +1,62 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
from zope.interface import implements
from twisted.application import service
+from twisted.internet import defer, task
+from twisted.python import log
from buildbot.interfaces import IChangeSource
from buildbot import util
class ChangeSource(service.Service, util.ComparableMixin):
implements(IChangeSource)
+ def describe(self):
+ return "no description"
+
+class PollingChangeSource(ChangeSource):
+ """
+ Utility subclass for ChangeSources that use some kind of periodic polling
+ operation. Subclasses should define C{poll} and set C{self.pollInterval}.
+ The rest is taken care of.
+ """
+
+ pollInterval = 60
+ "time (in seconds) between calls to C{poll}"
+
+ _loop = None
+ volatile = ['_loop'] # prevents Twisted from pickling this value
+
+ def poll(self):
+ """
+ Perform the polling operation, and return a deferred that will fire
+ when the operation is complete. Failures will be logged, but the
+ method will be called again after C{pollInterval} seconds.
+ """
+
+ def startService(self):
+ ChangeSource.startService(self)
+ def do_poll():
+ d = defer.maybeDeferred(self.poll)
+ d.addErrback(log.err)
+ return d
+ self._loop = task.LoopingCall(do_poll)
+ self._loop.start(self.pollInterval)
+
+ def stopService(self):
+ self._loop.stop()
+ return ChangeSource.stopService(self)
+
--- a/master/buildbot/changes/bonsaipoller.py
+++ b/master/buildbot/changes/bonsaipoller.py
@@ -1,15 +1,28 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import time
from xml.dom import minidom
-from twisted.python import log, failure
-from twisted.internet import reactor
-from twisted.internet.task import LoopingCall
-from twisted.web.client import getPage
+from twisted.python import log
+from twisted.web import client
from buildbot.changes import base, changes
class InvalidResultError(Exception):
def __init__(self, value="InvalidResultError"):
self.value = value
def __str__(self):
return repr(self.value)
@@ -182,108 +195,46 @@ class BonsaiParser:
raise InvalidResultError("Missing filename")
return filename
def _getRevision(self):
return self.currentFileNode.getAttribute("rev")
-class BonsaiPoller(base.ChangeSource):
- """This source will poll a bonsai server for changes and submit
- them to the change master."""
-
+class BonsaiPoller(base.PollingChangeSource):
compare_attrs = ["bonsaiURL", "pollInterval", "tree",
"module", "branch", "cvsroot"]
parent = None # filled in when we're added
- loop = None
- volatile = ['loop']
- working = False
def __init__(self, bonsaiURL, module, branch, tree="default",
cvsroot="/cvsroot", pollInterval=30, project=''):
- """
- @type bonsaiURL: string
- @param bonsaiURL: The base URL of the Bonsai server
- (ie. http://bonsai.mozilla.org)
- @type module: string
- @param module: The module to look for changes in. Commonly
- this is 'all'
- @type branch: string
- @param branch: The branch to look for changes in. This must
- match the
- 'branch' option for the Scheduler.
- @type tree: string
- @param tree: The tree to look for changes in. Commonly this
- is 'all'
- @type cvsroot: string
- @param cvsroot: The cvsroot of the repository. Usually this is
- '/cvsroot'
- @type pollInterval: int
- @param pollInterval: The time (in seconds) between queries for
- changes
-
- @type project: string
- @param project: project to attach to all Changes from this changesource
- """
-
self.bonsaiURL = bonsaiURL
self.module = module
self.branch = branch
self.tree = tree
self.cvsroot = cvsroot
self.repository = module != 'all' and module or ''
self.pollInterval = pollInterval
self.lastChange = time.time()
self.lastPoll = time.time()
- def startService(self):
- self.loop = LoopingCall(self.poll)
- base.ChangeSource.startService(self)
-
- reactor.callLater(0, self.loop.start, self.pollInterval)
-
- def stopService(self):
- self.loop.stop()
- return base.ChangeSource.stopService(self)
-
def describe(self):
str = ""
str += "Getting changes from the Bonsai service running at %s " \
% self.bonsaiURL
str += "<br>Using tree: %s, branch: %s, and module: %s" % (self.tree, \
self.branch, self.module)
return str
def poll(self):
- if self.working:
- log.msg("Not polling Bonsai because last poll is still working")
- else:
- self.working = True
- d = self._get_changes()
- d.addCallback(self._process_changes)
- d.addCallbacks(self._finished_ok, self._finished_failure)
- return
-
- def _finished_ok(self, res):
- assert self.working
- self.working = False
-
- # check for failure -- this is probably never hit but the twisted docs
- # are not clear enough to be sure. it is being kept "just in case"
- if isinstance(res, failure.Failure):
- log.msg("Bonsai poll failed: %s" % res)
- return res
-
- def _finished_failure(self, res):
- log.msg("Bonsai poll failed: %s" % res)
- assert self.working
- self.working = False
- return None # eat the failure
+ d = self._get_changes()
+ d.addCallback(self._process_changes)
+ return d
def _make_url(self):
args = ["treeid=%s" % self.tree, "module=%s" % self.module,
"branch=%s" % self.branch, "branchtype=match",
"sortby=Date", "date=explicit",
"mindate=%d" % self.lastChange,
"maxdate=%d" % int(time.time()),
"cvsroot=%s" % self.cvsroot, "xml=1"]
@@ -295,17 +246,17 @@ class BonsaiPoller(base.ChangeSource):
return url
def _get_changes(self):
url = self._make_url()
log.msg("Polling Bonsai tree at %s" % url)
self.lastPoll = time.time()
# get the page, in XML format
- return getPage(url, timeout=self.pollInterval)
+ return client.getPage(url, timeout=self.pollInterval)
def _process_changes(self, query):
try:
bp = BonsaiParser(query)
result = bp.getData()
except InvalidResultError, e:
log.msg("Could not process Bonsai query: " + e.value)
return
--- a/master/buildbot/changes/changes.py
+++ b/master/buildbot/changes/changes.py
@@ -1,9 +1,24 @@
-import sys, os, time
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import os, time
from cPickle import dump
from zope.interface import implements
from twisted.python import log, runtime
from twisted.web import html
from buildbot import interfaces, util
from buildbot.process.properties import Properties
deleted file mode 100644
--- a/master/buildbot/changes/freshcvs.py
+++ /dev/null
@@ -1,144 +0,0 @@
-
-import os.path
-
-from zope.interface import implements
-from twisted.cred import credentials
-from twisted.spread import pb
-from twisted.application.internet import TCPClient
-from twisted.python import log
-
-import cvstoys.common # to make sure VersionedPatch gets registered
-
-from buildbot.interfaces import IChangeSource
-from buildbot.pbutil import ReconnectingPBClientFactory
-from buildbot.changes.changes import Change
-from buildbot import util
-
-class FreshCVSListener(pb.Referenceable):
- def remote_notify(self, root, files, message, user):
- try:
- self.source.notify(root, files, message, user)
- except Exception:
- print "notify failed"
- log.err()
-
- def remote_goodbye(self, message):
- pass
-
-class FreshCVSConnectionFactory(ReconnectingPBClientFactory):
-
- def gotPerspective(self, perspective):
- log.msg("connected to FreshCVS daemon")
- ReconnectingPBClientFactory.gotPerspective(self, perspective)
- self.source.connected = True
- # TODO: freshcvs-1.0.10 doesn't handle setFilter correctly, it will
- # be fixed in the upcoming 1.0.11 . I haven't been able to test it
- # to make sure the failure mode is survivable, so I'll just leave
- # this out for now.
- return
- if self.source.prefix is not None:
- pathfilter = "^%s" % self.source.prefix
- d = perspective.callRemote("setFilter",
- None, pathfilter, None)
- # ignore failures, setFilter didn't work in 1.0.10 and this is
- # just an optimization anyway
- d.addErrback(lambda f: None)
-
- def clientConnectionLost(self, connector, reason):
- ReconnectingPBClientFactory.clientConnectionLost(self, connector,
- reason)
- self.source.connected = False
-
-class FreshCVSSourceNewcred(TCPClient, util.ComparableMixin):
- """This source will connect to a FreshCVS server associated with one or
- more CVS repositories. Each time a change is committed to a repository,
- the server will send us a message describing the change. This message is
- used to build a Change object, which is then submitted to the
- ChangeMaster.
-
- This class handles freshcvs daemons which use newcred. CVSToys-1.0.9
- does not, later versions might.
- """
-
- implements(IChangeSource)
- compare_attrs = ["host", "port", "username", "password", "prefix"]
-
- changemaster = None # filled in when we're added
- connected = False
-
- def __init__(self, host, port, user, passwd, prefix=None):
- self.host = host
- self.port = port
- self.username = user
- self.password = passwd
- if prefix is not None and not prefix.endswith("/"):
- log.msg("WARNING: prefix '%s' should probably end with a slash" \
- % prefix)
- self.prefix = prefix
- self.listener = l = FreshCVSListener()
- l.source = self
- self.factory = f = FreshCVSConnectionFactory()
- f.source = self
- self.creds = credentials.UsernamePassword(user, passwd)
- f.startLogin(self.creds, client=l)
- TCPClient.__init__(self, host, port, f)
-
- def __repr__(self):
- return "<FreshCVSSource where=%s, prefix=%s>" % \
- ((self.host, self.port), self.prefix)
-
- def describe(self):
- online = ""
- if not self.connected:
- online = " [OFFLINE]"
- return "freshcvs %s:%s%s" % (self.host, self.port, online)
-
- def notify(self, root, files, message, user):
- pathnames = []
- isdir = 0
- for f in files:
- if not isinstance(f, (cvstoys.common.VersionedPatch,
- cvstoys.common.Directory)):
- continue
- pathname, filename = f.pathname, f.filename
- #r1, r2 = getattr(f, 'r1', None), getattr(f, 'r2', None)
- if isinstance(f, cvstoys.common.Directory):
- isdir = 1
- path = os.path.join(pathname, filename)
- log.msg("FreshCVS notify '%s'" % path)
- if self.prefix:
- if path.startswith(self.prefix):
- path = path[len(self.prefix):]
- else:
- continue
- pathnames.append(path)
- if pathnames:
- # now() is close enough: FreshCVS *is* realtime, after all
- when=util.now()
- c = Change(user, pathnames, message, isdir, when=when)
- self.parent.addChange(c)
-
-class FreshCVSSourceOldcred(FreshCVSSourceNewcred):
- """This is for older freshcvs daemons (from CVSToys-1.0.9 and earlier).
- """
-
- def __init__(self, host, port, user, passwd,
- serviceName="cvstoys.notify", prefix=None):
- self.host = host
- self.port = port
- self.prefix = prefix
- self.listener = l = FreshCVSListener()
- l.source = self
- self.factory = f = FreshCVSConnectionFactory()
- f.source = self
- f.startGettingPerspective(user, passwd, serviceName, client=l)
- TCPClient.__init__(self, host, port, f)
-
- def __repr__(self):
- return "<FreshCVSSourceOldcred where=%s, prefix=%s>" % \
- ((self.host, self.port), self.prefix)
-
-# this is suitable for CVSToys-1.0.10 and later. If you run CVSToys-1.0.9 or
-# earlier, use FreshCVSSourceOldcred instead.
-FreshCVSSource = FreshCVSSourceNewcred
-
new file mode 100644
--- /dev/null
+++ b/master/buildbot/changes/gerritchangesource.py
@@ -0,0 +1,174 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.internet import reactor
+
+from buildbot.changes import base, changes
+from buildbot.util import json
+from buildbot import util
+from twisted.python import log
+from twisted.internet.protocol import ProcessProtocol
+
+class GerritChangeSource(base.ChangeSource):
+ """This source will maintain a connection to gerrit ssh server
+ that will provide us gerrit events in json format."""
+
+ compare_attrs = ["gerritserver", "gerritport"]
+
+ parent = None # filled in when we're added
+
+ STREAM_GOOD_CONNECTION_TIME = 120
+ "(seconds) connections longer than this are considered good, and reset the backoff timer"
+
+ STREAM_BACKOFF_MIN = 0.5
+ "(seconds) minimum, but nonzero, time to wait before retrying a failed connection"
+
+ STREAM_BACKOFF_EXPONENT = 1.5
+ "multiplier used to increase the backoff from MIN to MAX on repeated failures"
+
+ STREAM_BACKOFF_MAX = 60
+ "(seconds) maximum time to wait before retrying a failed connection"
+
+ def __init__(self, gerritserver, username, gerritport=29418):
+ """
+ @type gerritserver: string
+ @param gerritserver: the dns or ip that host the gerrit ssh server,
+
+ @type gerritport: int
+ @param gerritport: the port of the gerrit ssh server,
+
+ @type username: string
+ @param username: the username to use to connect to gerrit
+
+ """
+ # TODO: delete API comment when documented
+
+ self.gerritserver = gerritserver
+ self.gerritport = gerritport
+ self.username = username
+ self.process = None
+ self.streamProcessTimeout = self.STREAM_BACKOFF_MIN
+
+ class LocalPP(ProcessProtocol):
+ def __init__(self, change_source):
+ self.change_source = change_source
+ self.data = ""
+
+ def outReceived(self, data):
+ """Do line buffering."""
+ self.data += data
+ lines = self.data.split("\n")
+ self.data = lines.pop(-1) # last line is either empty or incomplete
+ for line in lines:
+ log.msg("gerrit: %s" % (line,))
+ self.change_source.lineReceived(line)
+
+ def errReceived(self, data):
+ log.msg("gerrit stderr: %s" % (data,))
+
+ def processEnded(self, status_object):
+ self.change_source.streamProcessStopped()
+
+ def lineReceived(self, line):
+ try:
+ event = json.loads(line)
+ except ValueError:
+ log.msg("bad json line: %s" % (line,))
+ return
+
+ if type(event) == type({}) and "type" in event and event["type"] in ["patchset-created", "ref-updated"]:
+ # flatten the event dictionary, for easy access with WithProperties
+ def flatten(event, base, d):
+ for k, v in d.items():
+ if type(v) == dict:
+ flatten(event, base + "." + k, v)
+ else: # already there
+ event[base + "." + k] = v
+
+ properties = {}
+ flatten(properties, "event", event)
+
+ if event["type"] == "patchset-created":
+ change = event["change"]
+ c = changes.Change(who="%s <%s>" % (change["owner"]["name"], change["owner"]["email"]),
+ project=change["project"],
+ branch=change["branch"],
+ revision=event["patchSet"]["revision"],
+ revlink=change["url"],
+ comments=change["subject"],
+ files=["unknown"],
+ category=event["type"],
+ properties=properties)
+ elif event["type"] == "ref-updated":
+ ref = event["refUpdate"]
+ c = changes.Change(who="%s <%s>" % (event["submitter"]["name"], event["submitter"]["email"]),
+ project=ref["project"],
+ branch=ref["refName"],
+ revision=ref["newRev"],
+ comments="Gerrit: patchset(s) merged.",
+ files=["unknown"],
+ category=event["type"],
+ properties=properties)
+ else:
+ return # this shouldn't happen anyway
+
+ self.parent.addChange(c)
+
+ def streamProcessStopped(self):
+ self.process = None
+
+ # if the service is stopped, don't try to restart
+ if not self.parent:
+ log.msg("service is not running; not reconnecting")
+ return
+
+ now = util.now()
+ if now - self.lastStreamProcessStart < self.STREAM_GOOD_CONNECTION_TIME:
+ # bad startup; start the stream process again after a timeout, and then
+ # increase the timeout
+ log.msg("'gerrit stream-events' failed; restarting after %ds" % round(self.streamProcessTimeout))
+ reactor.callLater(self.streamProcessTimeout, self.startStreamProcess)
+ self.streamProcessTimeout *= self.STREAM_BACKOFF_EXPONENT
+ if self.streamProcessTimeout > self.STREAM_BACKOFF_MAX:
+ self.streamProcessTimeout = self.STREAM_BACKOFF_MAX
+ else:
+ # good startup, but lost connection; restart immediately, and set the timeout
+ # to its minimum
+ self.startStreamProcess()
+ self.streamProcessTimeout = self.STREAM_BACKOFF_MIN
+
+ def startStreamProcess(self):
+ log.msg("starting 'gerrit stream-events'")
+ self.lastStreamProcessStart = util.now()
+ self.process = reactor.spawnProcess(self.LocalPP(self), "ssh", ["ssh", self.username+"@"+self.gerritserver,"-p", str(self.gerritport), "gerrit","stream-events"])
+
+ def startService(self):
+ self.startStreamProcess()
+
+ def stopService(self):
+ if self.process:
+ self.process.signalProcess("KILL")
+ # TODO: if this occurs while the process is restarting, some exceptions may
+ # be logged, although things will settle down normally
+ return base.ChangeSource.stopService(self)
+
+ def describe(self):
+ status = ""
+ if not self.process:
+ status = "[NOT CONNECTED - check log]"
+ str = ('GerritChangeSource watching the remote Gerrit repository %s@%s %s' %
+ (self.username, self.gerritserver, status))
+ return str
+
--- a/master/buildbot/changes/gitpoller.py
+++ b/master/buildbot/changes/gitpoller.py
@@ -1,276 +1,251 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import time
import tempfile
import os
import subprocess
-import select
-import errno
-
-from twisted.python import log, failure
-from twisted.internet import reactor, utils
-from twisted.internet.task import LoopingCall
-from twisted.web.client import getPage
+from twisted.python import log
+from twisted.internet import defer, utils
from buildbot.changes import base, changes
-class GitPoller(base.ChangeSource):
+class GitPoller(base.PollingChangeSource):
"""This source will poll a remote git repo for changes and submit
them to the change master."""
compare_attrs = ["repourl", "branch", "workdir",
- "pollinterval", "gitbin", "usetimestamps",
+ "pollInterval", "gitbin", "usetimestamps",
"category", "project"]
- parent = None # filled in when we're added
- loop = None
- volatile = ['loop']
- working = False
- running = False
-
def __init__(self, repourl, branch='master',
- workdir=None, pollinterval=10*60,
+ workdir=None, pollInterval=10*60,
gitbin='git', usetimestamps=True,
- category=None, project=''):
- """
- @type repourl: string
- @param repourl: the url that describes the remote repository,
- e.g. git@example.com:foobaz/myrepo.git
+ category=None, project=None,
+ pollinterval=-2):
+ # for backward compatibility; the parameter used to be spelled with 'i'
+ if pollinterval != -2:
+ pollInterval = pollinterval
+ if project is None: project = ''
- @type branch: string
- @param branch: the desired branch to fetch, will default to 'master'
-
- @type workdir: string
- @param workdir: the directory where the poller should keep its local repository.
- will default to <tempdir>/gitpoller_work
-
- @type pollinterval: int
- @param pollinterval: interval in seconds between polls, default is 10 minutes
-
- @type gitbin: string
- @param gitbin: path to the git binary, defaults to just 'git'
-
- @type usetimestamps: boolean
- @param usetimestamps: parse each revision's commit timestamp (default True), or
- ignore it in favor of the current time (to appear together
- in the waterfall page)
-
- @type category: string
- @param category: catergory associated with the change. Attached to
- the Change object produced by this changesource such that
- it can be targeted by change filters.
-
- @type project string
- @param project project that the changes are associated to. Attached to
- the Change object produced by this changesource such that
- it can be targeted by change filters.
- """
-
self.repourl = repourl
self.branch = branch
- self.pollinterval = pollinterval
+ self.pollInterval = pollInterval
self.lastChange = time.time()
self.lastPoll = time.time()
self.gitbin = gitbin
self.workdir = workdir
self.usetimestamps = usetimestamps
self.category = category
self.project = project
+ self.changeCount = 0
+ self.commitInfo = {}
if self.workdir == None:
self.workdir = tempfile.gettempdir() + '/gitpoller_work'
def startService(self):
- self.loop = LoopingCall(self.poll)
- base.ChangeSource.startService(self)
+ base.PollingChangeSource.startService(self)
- if not os.path.exists(self.workdir):
- log.msg('gitpoller: creating working dir %s' % self.workdir)
- os.makedirs(self.workdir)
+ dirpath = os.path.dirname(self.workdir.rstrip(os.sep))
+ if not os.path.exists(dirpath):
+ log.msg('gitpoller: creating parent directories for workdir')
+ os.makedirs(dirpath)
if not os.path.exists(self.workdir + r'/.git'):
log.msg('gitpoller: initializing working dir')
- os.system(self.gitbin + ' clone ' + self.repourl + ' ' + self.workdir)
-
- reactor.callLater(0, self.loop.start, self.pollinterval)
+ subprocess.check_call([self.gitbin, 'init', self.workdir])
+ subprocess.check_call([self.gitbin, 'remote', 'add', 'origin', self.repourl],
+ cwd=self.workdir)
+ subprocess.check_call([self.gitbin, 'fetch', 'origin'],
+ cwd=self.workdir)
+ if self.branch == 'master':
+ subprocess.check_call([self.gitbin, 'reset', '--hard',
+ 'origin/%s' % self.branch],
+ cwd=self.workdir)
+ else:
+ subprocess.check_call([self.gitbin, 'checkout', '-b', self.branch,
+ 'origin/%s' % self.branch],
+ cwd=self.workdir)
- self.running = True
-
- def stopService(self):
- if self.running:
- self.loop.stop()
- self.running = False
- return base.ChangeSource.stopService(self)
-
def describe(self):
status = ""
- if not self.running:
+ if not self.parent:
status = "[STOPPED - check log]"
str = 'GitPoller watching the remote git repository %s, branch: %s %s' \
% (self.repourl, self.branch, status)
return str
def poll(self):
- if self.working:
- log.msg('gitpoller: not polling git repo because last poll is still working')
- else:
- self.working = True
- d = self._get_changes()
- d.addCallback(self._process_changes)
- d.addCallbacks(self._changes_finished_ok, self._changes_finished_failure)
- d.addCallback(self._catch_up)
- d.addCallbacks(self._catch_up_finished_ok, self._catch_up__finished_failure)
- return
-
- def _get_git_output(self, args):
- git_args = [self.gitbin] + args
-
- p = subprocess.Popen(git_args,
- cwd=self.workdir,
- stdout=subprocess.PIPE)
-
- # dirty hack - work around EINTR oddness on Mac builder
- while True:
- try:
- output = p.communicate()[0]
- break
- except (OSError, select.error), e:
- if e[0] == errno.EINTR:
- continue
- else:
- raise
-
- if p.returncode != 0:
- raise EnvironmentError('call \'%s\' exited with error \'%s\', output: \'%s\'' %
- (args, p.returncode, output))
- return output
+ d = self._get_changes()
+ d.addCallback(self._process_changes)
+ d.addErrback(self._process_changes_failure)
+ d.addCallback(self._catch_up)
+ d.addErrback(self._catch_up_failure)
+ return d
def _get_commit_comments(self, rev):
args = ['log', rev, '--no-walk', r'--format=%s%n%b']
- output = self._get_git_output(args)
-
- if len(output.strip()) == 0:
- raise EnvironmentError('could not get commit comment for rev %s' % rev)
-
- return output
+ d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
+ d.addCallback(self._get_commit_comments_from_output)
+ return d
+
+ def _get_commit_comments_from_output(self,git_output):
+ stripped_output = git_output.strip()
+ if len(stripped_output) == 0:
+ raise EnvironmentError('could not get commit comment for rev')
+ self.commitInfo['comments'] = stripped_output
+ return self.commitInfo['comments'] # for tests
def _get_commit_timestamp(self, rev):
# unix timestamp
args = ['log', rev, '--no-walk', r'--format=%ct']
- output = self._get_git_output(args)
-
- try:
- stamp = float(output)
- except Exception, e:
- log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % output)
- raise e
-
- return stamp
-
+ d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
+ d.addCallback(self._get_commit_timestamp_from_output)
+ return d
+
+ def _get_commit_timestamp_from_output(self, git_output):
+ stripped_output = git_output.strip()
+ if self.usetimestamps:
+ try:
+ stamp = float(stripped_output)
+ except Exception, e:
+ log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % stripped_output)
+ raise e
+ self.commitInfo['timestamp'] = stamp
+ else:
+ self.commitInfo['timestamp'] = None
+ return self.commitInfo['timestamp'] # for tests
+
def _get_commit_files(self, rev):
args = ['log', rev, '--name-only', '--no-walk', r'--format=%n']
- fileList = self._get_git_output(args).split()
- return fileList
+ d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
+ d.addCallback(self._get_commit_files_from_output)
+ return d
+
+ def _get_commit_files_from_output(self, git_output):
+ fileList = git_output.split()
+ self.commitInfo['files'] = fileList
+ return self.commitInfo['files'] # for tests
def _get_commit_name(self, rev):
- args = ['log', rev, '--no-walk', r'--format=%cn']
- output = self._get_git_output(args)
-
- if len(output.strip()) == 0:
- raise EnvironmentError('could not get commit name for rev %s' % rev)
+ args = ['log', rev, '--no-walk', r'--format=%aE']
+ d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
+ d.addCallback(self._get_commit_name_from_output)
+ return d
- return output
+ def _get_commit_name_from_output(self, git_output):
+ stripped_output = git_output.strip()
+ if len(stripped_output) == 0:
+ raise EnvironmentError('could not get commit name for rev')
+ self.commitInfo['name'] = stripped_output
+ return self.commitInfo['name'] # for tests
def _get_changes(self):
log.msg('gitpoller: polling git repo at %s' % self.repourl)
self.lastPoll = time.time()
- # get a deferred object that performs the fetch
+ # get a deferred object that performs the git fetch
+
+ # This command always produces data on stderr, but we actually do not care
+ # about the stderr or stdout from this command. We set errortoo=True to
+ # avoid an errback from the deferred. The callback which will be added to this
+ # deferred will not use the response.
args = ['fetch', self.repourl, self.branch]
- d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env={}, errortoo=1 )
+ d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=True )
return d
- def _process_changes(self, res):
+ def _process_changes(self, unused_output):
# get the change list
revListArgs = ['log', 'HEAD..FETCH_HEAD', r'--format=%H']
- revs = self._get_git_output(revListArgs);
- revCount = 0
+ d = utils.getProcessOutput(self.gitbin, revListArgs, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
+ d.addCallback(self._process_changes_in_output)
+ return d
+
+ @defer.deferredGenerator
+ def _process_changes_in_output(self, git_output):
+ self.changeCount = 0
# process oldest change first
- revList = revs.split()
+ revList = git_output.split()
if revList:
revList.reverse()
- revCount = len(revList)
+ self.changeCount = len(revList)
- log.msg('gitpoller: processing %d changes' % revCount )
+ log.msg('gitpoller: processing %d changes: %s in "%s"' % (self.changeCount, revList, self.workdir) )
for rev in revList:
- if self.usetimestamps:
- commit_timestamp = self._get_commit_timestamp(rev)
- else:
- commit_timestamp = None # use current time
-
- c = changes.Change(who = self._get_commit_name(rev),
- revision = rev,
- files = self._get_commit_files(rev),
- comments = self._get_commit_comments(rev),
- when = commit_timestamp,
- branch = self.branch,
- category = self.category,
- project = self.project,
- repository = self.repourl)
- self.parent.addChange(c)
- self.lastChange = self.lastPoll
+ self.commitInfo = {}
+
+ deferreds = [
+ self._get_commit_timestamp(rev),
+ self._get_commit_name(rev),
+ self._get_commit_files(rev),
+ self._get_commit_comments(rev),
+ ]
+ dl = defer.DeferredList(deferreds)
+ dl.addCallback(self._add_change,rev)
+
+ # wait for that deferred to finish before starting the next
+ wfd = defer.waitForDeferred(dl)
+ yield wfd
+ wfd.getResult()
+
+
+ def _add_change(self, results, rev):
+ log.msg('gitpoller: _add_change results: "%s", rev: "%s" in "%s"' % (results, rev, self.workdir))
+
+ c = changes.Change(who=self.commitInfo['name'],
+ revision=rev,
+ files=self.commitInfo['files'],
+ comments=self.commitInfo['comments'],
+ when=self.commitInfo['timestamp'],
+ branch=self.branch,
+ category=self.category,
+ project=self.project,
+ repository=self.repourl)
+ log.msg('gitpoller: change "%s" in "%s"' % (c, self.workdir))
+ self.parent.addChange(c)
+ self.lastChange = self.lastPoll
- def _catch_up(self, res):
- log.msg('gitpoller: catching up to FETCH_HEAD')
-
- args = ['reset', '--hard', 'FETCH_HEAD']
- d = utils.getProcessOutputAndValue(self.gitbin, args, path=self.workdir, env={})
- return d;
- def _changes_finished_ok(self, res):
- assert self.working
- # check for failure -- this is probably never hit but the twisted docs
- # are not clear enough to be sure. it is being kept "just in case"
- if isinstance(res, failure.Failure):
- return self._changes_finished_failure(res)
-
- return res
-
- def _changes_finished_failure(self, res):
- log.msg('gitpoller: repo poll failed: %s' % res)
- assert self.working
- # eat the failure to continue along the defered chain
- # - we still want to catch up
+ def _process_changes_failure(self, f):
+ log.msg('gitpoller: repo poll failed')
+ log.err(f)
+ # eat the failure to continue along the defered chain - we still want to catch up
return None
- def _catch_up_finished_ok(self, res):
- assert self.working
-
- # check for failure -- this is probably never hit but the twisted docs
- # are not clear enough to be sure. it is being kept "just in case"
- if isinstance(res, failure.Failure):
- return self._catch_up__finished_failure(res)
-
- elif isinstance(res, tuple):
+ def _catch_up(self, res):
+ if self.changeCount == 0:
+ log.msg('gitpoller: no changes, no catch_up')
+ return
+ log.msg('gitpoller: catching up to FETCH_HEAD')
+ args = ['reset', '--hard', 'FETCH_HEAD']
+ d = utils.getProcessOutputAndValue(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']))
+ def convert_nonzero_to_failure(res):
(stdout, stderr, code) = res
if code != 0:
- e = EnvironmentError('catch up failed with exit code: %d' % code)
- return self._catch_up__finished_failure(failure.Failure(e))
-
- self.working = False
- return res
+ raise EnvironmentError('catch up failed with exit code: %d' % code)
+ d.addCallback(convert_nonzero_to_failure)
+ return d
- def _catch_up__finished_failure(self, res):
- assert self.working
- assert isinstance(res, failure.Failure)
- self.working = False
-
- log.msg('gitpoller: catch up failed: %s' % res)
- log.msg('gitpoller: stopping service - please resolve issues in local repo: %s' %
- self.workdir)
- self.stopService()
- return res
-
+ def _catch_up_failure(self, f):
+ log.err(f)
+ log.msg('gitpoller: please resolve issues in local repo: %s' % self.workdir)
+ # this used to stop the service, but this is (a) unfriendly to tests and (b)
+ # likely to leave the error message lost in a sea of other log messages
--- a/master/buildbot/changes/hgbuildbot.py
+++ b/master/buildbot/changes/hgbuildbot.py
@@ -1,14 +1,23 @@
-# hgbuildbot.py - mercurial hooks for buildbot
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# Copyright 2007 Frederic Leroy <fredo@starox.org>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# This software may be used and distributed according to the terms
-# of the GNU General Public License, incorporated herein by reference.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Portions Copyright Buildbot Team Members
+# Portions Copyright 2007 Frederic Leroy <fredo@starox.org>
# hook extension to send change notifications to buildbot when a changeset is
# brought into the repository from elsewhere.
#
# default mode is to use mercurial branch
#
# to use, configure hgbuildbot in .hg/hgrc like this:
#
@@ -90,17 +99,17 @@ def hook(ui, repo, hooktype, node=None,
if branch is None:
if branchtype is not None:
if branchtype == 'dirname':
branch = os.path.basename(repo.root)
if branchtype == 'inrepo':
branch = workingctx(repo).branch()
- s = sendchange.Sender(master, None)
+ s = sendchange.Sender(master)
d = defer.Deferred()
reactor.callLater(0, d.callback, None)
# process changesets
def _send(res, c):
if not fork:
ui.status("rev %s sent\n" % c['revision'])
return s.send(c['branch'], c['revision'], c['comments'],
c['files'], c['username'], category=category,
--- a/master/buildbot/changes/mail.py
+++ b/master/buildbot/changes/mail.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_mailparse -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
"""
Parse various kinds of 'CVS notify' email.
"""
import os, re
import time, calendar
import datetime
from email import message_from_file
@@ -45,305 +59,16 @@ class MaildirSource(MaildirService, util
self.parent.addChange(change)
os.rename(os.path.join(self.basedir, "new", filename),
os.path.join(self.basedir, "cur", filename))
def parse_file(self, fd, prefix=None):
m = message_from_file(fd)
return self.parse(m, prefix)
-class FCMaildirSource(MaildirSource):
- name = "FreshCVS"
-
- def parse(self, m, prefix=None):
- """Parse mail sent by FreshCVS"""
-
- # FreshCVS sets From: to "user CVS <user>", but the <> part may be
- # modified by the MTA (to include a local domain)
- name, addr = parseaddr(m["from"])
- if not name:
- return None # no From means this message isn't from FreshCVS
- cvs = name.find(" CVS")
- if cvs == -1:
- return None # this message isn't from FreshCVS
- who = name[:cvs]
-
- # we take the time of receipt as the time of checkin. Not correct,
- # but it avoids the out-of-order-changes issue. See the comment in
- # parseSyncmail about using the 'Date:' header
- when = util.now()
-
- files = []
- comments = ""
- isdir = 0
- lines = list(body_line_iterator(m))
- while lines:
- line = lines.pop(0)
- if line == "Modified files:\n":
- break
- while lines:
- line = lines.pop(0)
- if line == "\n":
- break
- line = line.rstrip("\n")
- linebits = line.split(None, 1)
- file = linebits[0]
- if prefix:
- # insist that the file start with the prefix: FreshCVS sends
- # changes we don't care about too
- if file.startswith(prefix):
- file = file[len(prefix):]
- else:
- continue
- if len(linebits) == 1:
- isdir = 1
- elif linebits[1] == "0 0":
- isdir = 1
- files.append(file)
- while lines:
- line = lines.pop(0)
- if line == "Log message:\n":
- break
- # message is terminated by "ViewCVS links:" or "Index:..." (patch)
- while lines:
- line = lines.pop(0)
- if line == "ViewCVS links:\n":
- break
- if line.find("Index: ") == 0:
- break
- comments += line
- comments = comments.rstrip() + "\n"
-
- if not files:
- return None
-
- change = changes.Change(who, files, comments, isdir, when=when)
-
- return change
-
-class SyncmailMaildirSource(MaildirSource):
- name = "Syncmail"
-
- def parse(self, m, prefix=None):
- """Parse messages sent by the 'syncmail' program, as suggested by the
- sourceforge.net CVS Admin documentation. Syncmail is maintained at
- syncmail.sf.net .
- """
- # pretty much the same as freshcvs mail, not surprising since CVS is
- # the one creating most of the text
-
- # The mail is sent from the person doing the checkin. Assume that the
- # local username is enough to identify them (this assumes a one-server
- # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
- # model)
- name, addr = parseaddr(m["from"])
- if not addr:
- return None # no From means this message isn't from FreshCVS
- at = addr.find("@")
- if at == -1:
- who = addr # might still be useful
- else:
- who = addr[:at]
-
- # we take the time of receipt as the time of checkin. Not correct (it
- # depends upon the email latency), but it avoids the
- # out-of-order-changes issue. Also syncmail doesn't give us anything
- # better to work with, unless you count pulling the v1-vs-v2
- # timestamp out of the diffs, which would be ugly. TODO: Pulling the
- # 'Date:' header from the mail is a possibility, and
- # email.Utils.parsedate_tz may be useful. It should be configurable,
- # however, because there are a lot of broken clocks out there.
- when = util.now()
-
- # calculate a "revision" based on that timestamp
- theCurrentTime = datetime.datetime.utcfromtimestamp(float(when))
- rev = theCurrentTime.strftime('%Y-%m-%d %H:%M:%S')
-
- subject = m["subject"]
- # syncmail puts the repository-relative directory in the subject:
- # mprefix + "%(dir)s %(file)s,%(oldversion)s,%(newversion)s", where
- # 'mprefix' is something that could be added by a mailing list
- # manager.
- # this is the only reasonable way to determine the directory name
- space = subject.find(" ")
- if space != -1:
- directory = subject[:space]
- else:
- directory = subject
-
- files = []
- comments = ""
- isdir = 0
- branch = None
-
- lines = list(body_line_iterator(m))
- while lines:
- line = lines.pop(0)
-
- if (line == "Modified Files:\n" or
- line == "Added Files:\n" or
- line == "Removed Files:\n"):
- break
-
- while lines:
- line = lines.pop(0)
- if line == "\n":
- break
- if line == "Log Message:\n":
- lines.insert(0, line)
- break
- line = line.lstrip()
- line = line.rstrip()
- # note: syncmail will send one email per directory involved in a
- # commit, with multiple files if they were in the same directory.
- # Unlike freshCVS, it makes no attempt to collect all related
- # commits into a single message.
-
- # note: syncmail will report a Tag underneath the ... Files: line
- # e.g.: Tag: BRANCH-DEVEL
-
- if line.startswith('Tag:'):
- branch = line.split(' ')[-1].rstrip()
- continue
-
- thesefiles = line.split(" ")
- for f in thesefiles:
- f = directory + "/" + f
- if prefix:
- # insist that the file start with the prefix: we may get
- # changes we don't care about too
- if f.startswith(prefix):
- f = f[len(prefix):]
- else:
- continue
- break
- # TODO: figure out how new directories are described, set
- # .isdir
- files.append(f)
-
- if not files:
- return None
-
- while lines:
- line = lines.pop(0)
- if line == "Log Message:\n":
- break
- # message is terminated by "Index:..." (patch) or "--- NEW FILE.."
- # or "--- filename DELETED ---". Sigh.
- while lines:
- line = lines.pop(0)
- if line.find("Index: ") == 0:
- break
- if re.search(r"^--- NEW FILE", line):
- break
- if re.search(r" DELETED ---$", line):
- break
- comments += line
- comments = comments.rstrip() + "\n"
-
- change = changes.Change(who, files, comments, isdir, when=when,
- branch=branch, revision=rev,
- category=self.category,
- repository=self.repository)
-
- return change
-
-# Bonsai mail parser by Stephen Davis.
-#
-# This handles changes for CVS repositories that are watched by Bonsai
-# (http://www.mozilla.org/bonsai.html)
-
-# A Bonsai-formatted email message looks like:
-#
-# C|1071099907|stephend|/cvs|Sources/Scripts/buildbot|bonsai.py|1.2|||18|7
-# A|1071099907|stephend|/cvs|Sources/Scripts/buildbot|master.cfg|1.1|||18|7
-# R|1071099907|stephend|/cvs|Sources/Scripts/buildbot|BuildMaster.py|||
-# LOGCOMMENT
-# Updated bonsai parser and switched master config to buildbot-0.4.1 style.
-#
-# :ENDLOGCOMMENT
-#
-# In the first example line, stephend is the user, /cvs the repository,
-# buildbot the directory, bonsai.py the file, 1.2 the revision, no sticky
-# and branch, 18 lines added and 7 removed. All of these fields might not be
-# present (during "removes" for example).
-#
-# There may be multiple "control" lines or even none (imports, directory
-# additions) but there is one email per directory. We only care about actual
-# changes since it is presumed directory additions don't actually affect the
-# build. At least one file should need to change (the makefile, say) to
-# actually make a new directory part of the build process. That's my story
-# and I'm sticking to it.
-
-class BonsaiMaildirSource(MaildirSource):
- name = "Bonsai"
-
- def parse(self, m, prefix=None):
- """Parse mail sent by the Bonsai cvs loginfo script."""
-
- # we don't care who the email came from b/c the cvs user is in the
- # msg text
-
- who = "unknown"
- timestamp = None
- files = []
- lines = list(body_line_iterator(m))
-
- # read the control lines (what/who/where/file/etc.)
- while lines:
- line = lines.pop(0)
- if line == "LOGCOMMENT\n":
- break;
- line = line.rstrip("\n")
-
- # we'd like to do the following but it won't work if the number of
- # items doesn't match so...
- # what, timestamp, user, repo, module, file = line.split( '|' )
- items = line.split('|')
- if len(items) < 6:
- # not a valid line, assume this isn't a bonsai message
- return None
-
- try:
- # just grab the bottom-most timestamp, they're probably all the
- # same. TODO: I'm assuming this is relative to the epoch, but
- # this needs testing.
- timestamp = int(items[1])
- except ValueError:
- pass
-
- user = items[2]
- if user:
- who = user
-
- module = items[4]
- file = items[5]
- if module and file:
- path = "%s/%s" % (module, file)
- files.append(path)
- sticky = items[7]
- branch = items[8]
-
- # if no files changed, return nothing
- if not files:
- return None
-
- # read the comments
- comments = ""
- while lines:
- line = lines.pop(0)
- if line == ":ENDLOGCOMMENT\n":
- break
- comments += line
- comments = comments.rstrip() + "\n"
-
- # return buildbot Change object
- return changes.Change(who, files, comments, when=timestamp,
- branch=branch)
-
class CVSMaildirSource(MaildirSource):
name = "CVSMaildirSource"
def __init__(self, maildir, prefix=None, category='',
repository='', urlmaker=None, properties={}):
"""If urlmaker is defined, it will be called with three arguments:
filename, previous version, new version. It returns a url for that
file."""
@@ -419,33 +144,35 @@ class CVSMaildirSource(MaildirSource):
cvsmode = m.group(1)
continue
m = filesRE.match(line)
if m:
fileList = m.group(1)
continue
m = modRE.match(line)
if m:
- module = m.group(1)
+ # We don't actually use this
+ #module = m.group(1)
continue
m = pathRE.match(line)
if m:
path = m.group(1)
continue
m = projRE.match(line)
if m:
project = m.group(1)
continue
m = tagRE.match(line)
if m:
branch = m.group(1)
continue
m = updateRE.match(line)
if m:
- updateof = m.group(1)
+ # We don't actually use this
+ #updateof = m.group(1)
continue
if line == "Log Message:\n":
break
# CVS 1.11 lists files as:
# repo/path file,old-version,new-version file2,old-version,new-version
# Version 1.12 lists files as:
# file1 old-version new-version file2 old-version new-version
@@ -571,17 +298,16 @@ class SVNCommitEmailMaildirSource(Maildi
# timestamp out of the diffs, which would be ugly. TODO: Pulling the
# 'Date:' header from the mail is a possibility, and
# email.Utils.parsedate_tz may be useful. It should be configurable,
# however, because there are a lot of broken clocks out there.
when = util.now()
files = []
comments = ""
- isdir = 0
lines = list(body_line_iterator(m))
rev = None
while lines:
line = lines.pop(0)
# "Author: jmason"
match = re.search(r"^Author: (\S+)", line)
if match:
--- a/master/buildbot/changes/maildir.py
+++ b/master/buildbot/changes/maildir.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# This is a class which watches a maildir for new messages. It uses the
# linux dirwatcher API (if available) to look for new files. The
# .messageReceived method is invoked with the filename of the new message,
# relative to the top of the maildir (so it will look like "new/blahblah").
import os
from twisted.python import log
--- a/master/buildbot/changes/manager.py
+++ b/master/buildbot/changes/manager.py
@@ -1,44 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
import time
from zope.interface import implements
from twisted.python import log
from twisted.internet import defer
from twisted.application import service
@@ -67,20 +45,16 @@ class ChangeManager(service.MultiService
There are several different variants of the second type of source:
- L{buildbot.changes.mail.MaildirSource} watches a maildir for CVS
commit mail. It uses DNotify if available, or polls every 10
seconds if not. It parses incoming mail to determine what files
were changed.
- - L{buildbot.changes.freshcvs.FreshCVSSource} makes a PB
- connection to the CVSToys 'freshcvs' daemon and relays any
- changes it announces.
-
"""
implements(interfaces.IEventSource)
changeHorizon = None
lastPruneChanges = None
name = "changemanager"
--- a/master/buildbot/changes/p4poller.py
+++ b/master/buildbot/changes/p4poller.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_p4poller -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# Many thanks to Dave Peticolas for contributing this module
import re
import time
import os
from twisted.python import log
--- a/master/buildbot/changes/pb.py
+++ b/master/buildbot/changes/pb.py
@@ -1,11 +1,26 @@
-# -*- test-case-name: buildbot.test.test_changes -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.python import log
+from twisted.internet import defer
from buildbot.pbutil import NewCredPerspective
from buildbot.changes import base, changes
class ChangePerspective(NewCredPerspective):
def __init__(self, changemaster, prefix):
self.changemaster = changemaster
@@ -38,20 +53,20 @@ class ChangePerspective(NewCredPerspecti
when=changedict.get('when'),
properties=changedict.get('properties', {}),
repository=changedict.get('repository', '') or '',
project=changedict.get('project', '') or '',
)
self.changemaster.addChange(change)
class PBChangeSource(base.ChangeSource):
- compare_attrs = ["user", "passwd", "port", "prefix"]
+ compare_attrs = ["user", "passwd", "port", "prefix", "port"]
def __init__(self, user="change", passwd="changepw", port=None,
- prefix=None, sep=None):
+ prefix=None, sep=None):
"""I listen on a TCP port for Changes from 'buildbot sendchange'.
I am a ChangeSource which will accept Changes from a remote source. I
share a TCP listening port with the buildslaves.
The 'buildbot sendchange' command, the contrib/svn_buildbot.py tool,
and the contrib/bzr_buildbot.py tool know how to send changes to me.
@@ -67,46 +82,51 @@ class PBChangeSource(base.ChangeSource):
follow one branch and to get correct tree-relative
filenames.
@param sep: DEPRECATED (with an axe). sep= was removed in
buildbot-0.7.4 . Instead of using it, you should use
prefix= with a trailing directory separator. This
docstring (and the better-than-nothing error message
which occurs when you use it) will be removed in 0.7.5 .
+
+ @param port: strport to use, or None to use the master's slavePortnum
"""
# sep= was removed in 0.7.4 . This more-helpful-than-nothing error
# message will be removed in 0.7.5 .
assert sep is None, "prefix= is now a complete string, do not use sep="
- # TODO: current limitations
- assert user == "change"
- assert passwd == "changepw"
- assert port == None
+
self.user = user
self.passwd = passwd
self.port = port
self.prefix = prefix
+ self.registration = None
def describe(self):
# TODO: when the dispatcher is fixed, report the specific port
#d = "PB listener on port %d" % self.port
d = "PBChangeSource listener on all-purpose slaveport"
if self.prefix is not None:
d += " (prefix '%s')" % self.prefix
return d
def startService(self):
base.ChangeSource.startService(self)
- # our parent is the ChangeMaster object
- # find the master's Dispatch object and register our username
- # TODO: the passwd should be registered here too
master = self.parent.parent
- master.dispatcher.register(self.user, self)
+ port = self.port
+ if not port:
+ port = master.slavePortnum
+ self.registration = master.pbmanager.register(
+ port, self.user, self.passwd,
+ self.getPerspective)
def stopService(self):
- base.ChangeSource.stopService(self)
- # unregister our username
- master = self.parent.parent
- master.dispatcher.unregister(self.user)
+ d = defer.maybeDeferred(base.ChangeSource.stopService, self)
+ def unreg(_):
+ if self.registration:
+ return self.registration.unregister()
+ d.addCallback(unreg)
+ return d
- def getPerspective(self):
+ def getPerspective(self, mind, username):
+ assert username == self.user
return ChangePerspective(self.parent, self.prefix)
--- a/master/buildbot/changes/svnpoller.py
+++ b/master/buildbot/changes/svnpoller.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_svnpoller -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# Based on the work of Dave Peticolas for the P4poll
# Changed to svn (using xml.dom.minidom) by Niklaus Giger
# Hacked beyond recognition by Brian Warner
from twisted.python import log
from twisted.internet import defer, reactor, utils
from twisted.internet.task import LoopingCall
@@ -50,17 +64,17 @@ class SVNPoller(base.ChangeSource, util.
last_change = None
loop = None
working = False
def __init__(self, svnurl, split_file=None,
svnuser=None, svnpasswd=None,
pollinterval=10*60, histmax=100,
svnbin='svn', revlinktmpl='', category=None,
- project=None, cachepath=None):
+ project='', cachepath=None):
"""
@type svnurl: string
@param svnurl: the SVN URL that describes the repository and
subdirectory to watch. If this ChangeSource should
only pay attention to a single branch, this should
point at the repository for that branch, like
svn://svn.twistedmatrix.com/svn/Twisted/trunk . If it
should follow multiple branches, point it at the
@@ -82,18 +96,18 @@ class SVNPoller(base.ChangeSource, util.
(BRANCH, FILEPATH). This function should match
your repository's branch-naming policy. Each
changed file has a fully-qualified URL that can be
split into a prefix (which equals the value of the
'svnurl' argument) and a suffix; it is this suffix
which is passed to the split_file function.
If the function returns None, the file is ignored.
- Use this to indicate that the file is not a part
- of this project.
+ Use this to indicate that the file is not relevant
+ to this buildmaster.
For example, if your repository puts the trunk in
trunk/... and branches are in places like
branches/1.5/..., your split_file function could
look like the following (this function is
available as svnpoller.split_file_branches)::
pieces = path.split('/')
@@ -115,19 +129,19 @@ class SVNPoller(base.ChangeSource, util.
branch = None
pieces.pop(0) # remove 'trunk'
elif pieces[0] == 'branches':
pieces.pop(0) # remove 'branches'
# grab branch name
branch = 'branches/' + pieces.pop(0)
else:
return None # something weird
- projectname = pieces.pop(0)
- if projectname != 'ProjectA':
- return None # wrong project
+ productname = pieces.pop(0)
+ if productname != 'ProjectA':
+ return None # wrong product
return (branch, '/'.join(pieces))
The default of split_file= is None, which
indicates that no splitting should be done. This
is equivalent to the following function::
return (None, path)
--- a/master/buildbot/clients/base.py
+++ b/master/buildbot/clients/base.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import re
from twisted.spread import pb
from twisted.cred import credentials, error
from twisted.internet import reactor
class StatusClient(pb.Referenceable):
--- a/master/buildbot/clients/debug.py
+++ b/master/buildbot/clients/debug.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.internet import gtk2reactor
gtk2reactor.install()
from twisted.internet import reactor
from twisted.python import util
from twisted.spread import pb
from twisted.cred import credentials
import gtk.glade #@UnresolvedImport
--- a/master/buildbot/clients/gtkPanes.py
+++ b/master/buildbot/clients/gtkPanes.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.internet import gtk2reactor
gtk2reactor.install() #@UndefinedVariable
import sys, time
import pygtk #@UnresolvedImport
pygtk.require("2.0")
--- a/master/buildbot/clients/sendchange.py
+++ b/master/buildbot/clients/sendchange.py
@@ -1,32 +1,45 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.spread import pb
from twisted.cred import credentials
from twisted.internet import reactor
class Sender:
- def __init__(self, master, user=None):
- self.user = user
+ def __init__(self, master, auth=('change','changepw')):
+ self.username, self.password = auth
self.host, self.port = master.split(":")
self.port = int(self.port)
self.num_changes = 0
- def send(self, branch, revision, comments, files, user=None, category=None,
+ def send(self, branch, revision, comments, files, who=None, category=None,
when=None, properties={}, repository='', project='', revlink=''):
- if user is None:
- user = self.user
- change = {'project': project, 'repository': repository, 'who': user,
+ change = {'project': project, 'repository': repository, 'who': who,
'files': files, 'comments': comments, 'branch': branch,
'revision': revision, 'category': category, 'when': when,
'properties': properties, 'revlink': revlink}
self.num_changes += 1
f = pb.PBClientFactory()
- d = f.login(credentials.UsernamePassword("change", "changepw"))
+ d = f.login(credentials.UsernamePassword(self.username, self.password))
reactor.connectTCP(self.host, self.port, f)
d.addCallback(self.addChange, change)
return d
def addChange(self, remote, change):
d = remote.callRemote('addChange', change)
d.addCallback(lambda res: remote.broker.transport.loseConnection())
return d
--- a/master/buildbot/clients/tryclient.py
+++ b/master/buildbot/clients/tryclient.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_scheduler,buildbot.test.test_vc -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import sys, os, re, time, random
from twisted.internet import utils, protocol, defer, reactor, task
from twisted.spread import pb
from twisted.cred import credentials
from twisted.python import log
from twisted.python.procutils import which
--- a/master/buildbot/config.py
+++ b/master/buildbot/config.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.util import safeTranslate
class BuilderConfig:
"""
Used in config files to specify a builder - this can be subclassed by users
to add extra config args, set defaults, or whatever. It is converted to a
--- a/master/buildbot/db/__init__.py
+++ b/master/buildbot/db/__init__.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# garbage-collection rules: the following rows can be GCed:
# a patch that isn't referenced by any sourcestamps
# a sourcestamp that isn't referenced by any buildsets
# a buildrequest that isn't referenced by any buildsets
# a buildset which is complete and isn't referenced by anything in
# scheduler_upstream_buildsets
# a scheduler_upstream_buildsets row that is not active
# a build that references a non-existent buildrequest
--- a/master/buildbot/db/connector.py
+++ b/master/buildbot/db/connector.py
@@ -1,46 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
-# Chris AtLee <catlee@mozilla.com>
-# Dustin Mitchell <dustin@zmanda.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
import sys, collections, base64
from twisted.python import log, threadable
from twisted.internet import defer
from twisted.enterprise import adbapi
from buildbot import util
from buildbot.util import collections as bbcollections
@@ -266,21 +242,19 @@ class DBConnector(util.ComparableMixin):
eventually(observer, category, *args)
def subscribe_to(self, category, observer):
self._subscribers[category].add(observer)
def runQuery(self, *args, **kwargs):
assert self._started
self._pending_operation_count += 1
- start = self._getCurrentTime()
- #t = self._start_operation() # why is this commented out? -warner
d = self._pool.runQuery(*args, **kwargs)
- #d.addBoth(self._runQuery_done, start, t)
return d
+
def _runQuery_done(self, res, start, t):
self._end_operation(t)
self._add_query_time(start)
self._pending_operation_count -= 1
return res
def _add_query_time(self, start):
elapsed = self._getCurrentTime() - start
--- a/master/buildbot/db/dbspec.py
+++ b/master/buildbot/db/dbspec.py
@@ -1,46 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
-# Chris AtLee <catlee@mozilla.com>
-# Dustin Mitchell <dustin@zmanda.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
import sys, os, cgi, re, time
from twisted.python import log, reflect
from twisted.enterprise import adbapi
from buildbot import util
@@ -215,21 +191,23 @@ class DBSpec(object):
raise ValueError("Unsupported dbapi %s" % driver)
def _get_sqlite_dbapi_name(self):
# see which dbapi we can use and return that name; prefer
# pysqlite2.dbapi2 if it is available.
sqlite_dbapi_name = None
try:
from pysqlite2 import dbapi2 as sqlite3
+ assert sqlite3
sqlite_dbapi_name = "pysqlite2.dbapi2"
except ImportError:
# don't use built-in sqlite3 on 2.5 -- it has *bad* bugs
if sys.version_info >= (2,6):
import sqlite3
+ assert sqlite3
sqlite_dbapi_name = "sqlite3"
else:
raise
return sqlite_dbapi_name
def get_dbapi(self):
"""
Get the dbapi module used for this connection (for things like
--- a/master/buildbot/db/exceptions.py
+++ b/master/buildbot/db/exceptions.py
@@ -1,44 +1,20 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
-# Chris AtLee <catlee@mozilla.com>
-# Dustin Mitchell <dustin@zmanda.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
class DBAlreadyExistsError(Exception):
pass
class DatabaseNotReadyError(Exception):
pass
--- a/master/buildbot/db/schema/base.py
+++ b/master/buildbot/db/schema/base.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
class Upgrader(object):
def __init__(self, dbapi, conn, basedir, quiet=False):
self.dbapi = dbapi
self.conn = conn
self.basedir = basedir
self.quiet = quiet
--- a/master/buildbot/db/schema/manager.py
+++ b/master/buildbot/db/schema/manager.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.python import reflect
# note that schema modules are not loaded unless an upgrade is taking place
CURRENT_VERSION = 6
class DBSchemaManager(object):
"""
--- a/master/buildbot/db/schema/v1.py
+++ b/master/buildbot/db/schema/v1.py
@@ -1,45 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
-# Chris AtLee <catlee@mozilla.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
import cPickle
import textwrap
import os
import sys
from twisted.persisted import styles
--- a/master/buildbot/db/schema/v2.py
+++ b/master/buildbot/db/schema/v2.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.db.schema import base
class Upgrader(base.Upgrader):
def upgrade(self):
self.add_columns()
self.set_version()
--- a/master/buildbot/db/schema/v3.py
+++ b/master/buildbot/db/schema/v3.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.db.schema import base
class Upgrader(base.Upgrader):
def upgrade(self):
self.migrate_schedulers()
self.set_version()
def migrate_schedulers(self):
--- a/master/buildbot/db/schema/v4.py
+++ b/master/buildbot/db/schema/v4.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.db.schema import base
class Upgrader(base.Upgrader):
def upgrade(self):
self.migrate_buildrequests()
self.migrate_builds()
self.migrate_buildsets()
self.migrate_changes()
@@ -14,67 +29,70 @@ class Upgrader(base.Upgrader):
def makeAutoincColumn(self, name):
if self.dbapiName == 'MySQLdb':
return "`%s` INTEGER PRIMARY KEY AUTO_INCREMENT" % name
elif self.dbapiName in ('sqlite3', 'pysqlite2.dbapi2'):
return "`%s` INTEGER PRIMARY KEY AUTOINCREMENT" % name
raise ValueError("Unsupported dbapi: %s" % self.dbapiName)
def migrate_table(self, table_name, schema):
- old_name = "%s_old" % table_name
+ names = {
+ 'old_name': "%s_old" % table_name,
+ 'table_name': table_name,
+ }
cursor = self.conn.cursor()
# If this fails, there's no cleaning up to do
cursor.execute("""
ALTER TABLE %(table_name)s
RENAME TO %(old_name)s
- """ % locals())
+ """ % names)
try:
cursor.execute(schema)
except:
# Restore the original table
cursor.execute("""
ALTER TABLE %(old_name)s
RENAME TO %(table_name)s
- """ % locals())
+ """ % names)
raise
try:
cursor.execute("""
INSERT INTO %(table_name)s
SELECT * FROM %(old_name)s
- """ % locals())
+ """ % names)
cursor.execute("""
DROP TABLE %(old_name)s
- """ % locals())
+ """ % names)
except:
# Clean up the new table, and restore the original
cursor.execute("""
DROP TABLE %(table_name)s
- """ % locals())
+ """ % names)
cursor.execute("""
ALTER TABLE %(old_name)s
RENAME TO %(table_name)s
- """ % locals())
+ """ % names)
raise
def set_version(self):
c = self.conn.cursor()
c.execute("""UPDATE version set version = 4 where version = 3""")
def migrate_schedulers(self):
schedulerid_col = self.makeAutoincColumn('schedulerid')
schema = """
CREATE TABLE schedulers (
%(schedulerid_col)s, -- joins to other tables
`name` VARCHAR(100) NOT NULL, -- the scheduler's name according to master.cfg
`class_name` VARCHAR(100) NOT NULL, -- the scheduler's class
`state` VARCHAR(1024) NOT NULL -- JSON-encoded state dictionary
);
- """ % locals()
+ """ % {'schedulerid_col': schedulerid_col}
self.migrate_table('schedulers', schema)
# Fix up indices
cursor = self.conn.cursor()
cursor.execute("""
CREATE UNIQUE INDEX `name_and_class` ON
schedulers (`name`, `class_name`)
""")
@@ -85,17 +103,17 @@ class Upgrader(base.Upgrader):
CREATE TABLE builds (
%(buildid_col)s,
`number` INTEGER NOT NULL, -- BuilderStatus.getBuild(number)
-- 'number' is scoped to both the local buildmaster and the buildername
`brid` INTEGER NOT NULL, -- matches buildrequests.id
`start_time` INTEGER NOT NULL,
`finish_time` INTEGER
);
- """ % locals()
+ """ % {'buildid_col': buildid_col}
self.migrate_table('builds', schema)
def migrate_changes(self):
changeid_col = self.makeAutoincColumn('changeid')
schema = """
CREATE TABLE changes (
%(changeid_col)s, -- also serves as 'change number'
`author` VARCHAR(1024) NOT NULL,
@@ -110,17 +128,17 @@ class Upgrader(base.Upgrader):
-- repository specifies, along with revision and branch, the
-- source tree in which this change was detected.
`repository` TEXT NOT NULL default '',
-- project names the project this source code represents. It is used
-- later to filter changes
`project` TEXT NOT NULL default ''
);
- """ % locals()
+ """ % {'changeid_col': changeid_col}
self.migrate_table('changes', schema)
# Drop changes_nextid columnt
cursor = self.conn.cursor()
cursor.execute("DROP TABLE changes_nextid")
def migrate_buildrequests(self):
buildrequestid_col = self.makeAutoincColumn('id')
@@ -154,53 +172,53 @@ class Upgrader(base.Upgrader):
-- results is only valid when complete==1
`results` SMALLINT, -- 0=SUCCESS,1=WARNINGS,etc, from status/builder.py
`submitted_at` INTEGER NOT NULL,
`complete_at` INTEGER
);
- """ % locals()
+ """ % {'buildrequestid_col': buildrequestid_col}
self.migrate_table('buildrequests', schema)
def migrate_buildsets(self):
buildsetsid_col = self.makeAutoincColumn('id')
schema = """
CREATE TABLE buildsets (
%(buildsetsid_col)s,
`external_idstring` VARCHAR(256),
`reason` VARCHAR(256),
`sourcestampid` INTEGER NOT NULL,
`submitted_at` INTEGER NOT NULL,
`complete` SMALLINT NOT NULL default 0,
`complete_at` INTEGER,
`results` SMALLINT -- 0=SUCCESS,2=FAILURE, from status/builder.py
-- results is NULL until complete==1
);
- """ % locals()
+ """ % {'buildsetsid_col': buildsetsid_col}
self.migrate_table("buildsets", schema)
def migrate_patches(self):
patchesid_col = self.makeAutoincColumn('id')
schema = """
CREATE TABLE patches (
%(patchesid_col)s,
`patchlevel` INTEGER NOT NULL,
`patch_base64` TEXT NOT NULL, -- encoded bytestring
`subdir` TEXT -- usually NULL
);
- """ % locals()
+ """ % {'patchesid_col': patchesid_col}
self.migrate_table("patches", schema)
def migrate_sourcestamps(self):
sourcestampsid_col = self.makeAutoincColumn('id')
schema = """
CREATE TABLE sourcestamps (
%(sourcestampsid_col)s,
`branch` VARCHAR(256) default NULL,
`revision` VARCHAR(256) default NULL,
`patchid` INTEGER default NULL,
`repository` TEXT not null default '',
`project` TEXT not null default ''
);
- """ % locals()
+ """ % {'sourcestampsid_col': sourcestampsid_col}
self.migrate_table("sourcestamps", schema)
--- a/master/buildbot/db/schema/v5.py
+++ b/master/buildbot/db/schema/v5.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.db.schema import base
class Upgrader(base.Upgrader):
def upgrade(self):
self.add_index("buildrequests", "buildsetid")
self.add_index("buildrequests", "buildername", 255)
self.add_index("buildrequests", "complete")
self.add_index("buildrequests", "claimed_at")
@@ -42,13 +57,13 @@ class Upgrader(base.Upgrader):
self.set_version()
def add_index(self, table, column, length=None):
lengthstr=""
if length is not None and self.dbapiName == 'MySQLdb':
lengthstr = " (%i)" % length
q = "CREATE INDEX `%(table)s_%(column)s` ON `%(table)s` (`%(column)s`%(lengthstr)s)"
cursor = self.conn.cursor()
- cursor.execute(q % locals())
+ cursor.execute(q % {'table': table, 'column': column, 'lengthstr': lengthstr})
def set_version(self):
c = self.conn.cursor()
c.execute("""UPDATE version set version = 5 where version = 4""")
--- a/master/buildbot/db/schema/v6.py
+++ b/master/buildbot/db/schema/v6.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.db.schema import base
class Upgrader(base.Upgrader):
def upgrade(self):
cursor = self.conn.cursor()
cursor.execute("DROP table last_access")
cursor.execute("""UPDATE version set version = 6 where version = 5""")
--- a/master/buildbot/db/util.py
+++ b/master/buildbot/db/util.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
def sql_insert(dbapi, table, columns):
"""
Make an SQL insert statement for the given table and columns, using the
appropriate paramstyle for the dbi. Note that this only supports positional
parameters. This will need to be reworked if Buildbot supports a backend with
a name-based paramstyle.
"""
--- a/master/buildbot/ec2buildslave.py
+++ b/master/buildbot/ec2buildslave.py
@@ -1,15 +1,29 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Portions Copyright Buildbot Team Members
+# Portions Copyright Canonical Ltd. 2009
+
"""A LatentSlave that uses EC2 to instantiate the slaves on demand.
Tested with Python boto 1.5c
"""
-# Portions copyright Canonical Ltd. 2009
-
import os
import re
import time
import boto
import boto.exception
from twisted.internet import defer, threads
from twisted.python import log
@@ -99,16 +113,17 @@ class EC2LatentBuildSlave(AbstractLatent
# We currently discard the keypair data because we don't need it.
# If we do need it in the future, we will always recreate the keypairs
# because there is no way to
# programmatically retrieve the private key component, unless we
# generate it and store it on the filesystem, which is an unnecessary
# usage requirement.
try:
key_pair = self.conn.get_all_key_pairs(keypair_name)[0]
+ assert key_pair
# key_pair.delete() # would be used to recreate
except boto.exception.EC2ResponseError, e:
if 'InvalidKeyPair.NotFound' not in e.body:
if 'AuthFailure' in e.body:
print ('POSSIBLE CAUSES OF ERROR:\n'
' Did you sign up for EC2?\n'
' Did you put a credit card number in your AWS '
'account?\n'
@@ -117,16 +132,17 @@ class EC2LatentBuildSlave(AbstractLatent
# make one; we would always do this, and stash the result, if we
# needed the key (for instance, to SSH to the box). We'd then
# use paramiko to use the key to connect.
self.conn.create_key_pair(keypair_name)
# create security group
try:
group = self.conn.get_all_security_groups(security_name)[0]
+ assert group
except boto.exception.EC2ResponseError, e:
if 'InvalidGroup.NotFound' in e.body:
self.security_group = self.conn.create_security_group(
security_name,
'Authorization to access the buildbot instance.')
# Authorize the master as necessary
# TODO this is where we'd open the hole to do the reverse pb
# connect to the buildbot
@@ -138,16 +154,17 @@ class EC2LatentBuildSlave(AbstractLatent
raise
# get the image
if self.ami is not None:
self.image = self.conn.get_image(self.ami)
else:
# verify we have access to at least one acceptable image
discard = self.get_image()
+ assert discard
# get the specified elastic IP, if any
if elastic_ip is not None:
elastic_ip = self.conn.get_all_addresses([elastic_ip])[0]
self.elastic_ip = elastic_ip
def get_image(self):
if self.image is not None:
--- a/master/buildbot/interfaces.py
+++ b/master/buildbot/interfaces.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
"""Interface documentation.
Define the interfaces that are implemented by various buildbot classes.
"""
# E0211: Method has no argument
# E0213: Method should have "self" as first argument
# pylint: disable-msg=E0211,E0213
@@ -29,24 +44,16 @@ class IChangeSource(Interface):
"""Object which feeds Change objects to the changemaster. When files or
directories are changed and the version control system provides some
kind of notification, this object should turn it into a Change object
and pass it through::
self.changemaster.addChange(change)
"""
- def start():
- """Called when the buildmaster starts. Can be used to establish
- connections to VC daemons or begin polling."""
-
- def stop():
- """Called when the buildmaster shuts down. Connections should be
- terminated, polling timers should be canceled."""
-
def describe():
"""Should return a string which briefly describes this source. This
string will be displayed in an HTML status page."""
class IScheduler(Interface):
"""I watch for Changes in the source tree and decide when to trigger
Builds. I create BuildSet objects and submit them to the BuildMaster. I
am a service, and the BuildMaster is always my parent.
@@ -122,17 +129,17 @@ class ISourceStamp(Interface):
def canBeMergedWith(self, other):
"""
Can this SourceStamp be merged with OTHER?
"""
def mergeWith(self, others):
"""Generate a SourceStamp for the merger of me and all the other
- BuildRequests. This is called by a Build when it starts, to figure
+ SourceStamps. This is called by a Build when it starts, to figure
out what its sourceStamp should be."""
def getAbsoluteSourceStamp(self, got_revision):
"""Get a new SourceStamp object reflecting the actual revision found
by a Source step."""
def getText(self):
"""Returns a list of strings to describe the stamp. These are
--- a/master/buildbot/libvirtbuildslave.py
+++ b/master/buildbot/libvirtbuildslave.py
@@ -1,9 +1,23 @@
-# Copyright 2010 Isotoma Limited
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Portions Copyright Buildbot Team Members
+# Portions Copyright 2010 Isotoma Limited
import os
from twisted.internet import defer, utils, reactor, threads
from twisted.python import log
from buildbot.buildslave import AbstractBuildSlave, AbstractLatentBuildSlave
import libvirt
--- a/master/buildbot/locks.py
+++ b/master/buildbot/locks.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_locks -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.python import log
from twisted.internet import reactor, defer
from buildbot import util
if False: # for debugging
debuglog = log.msg
else:
--- a/master/buildbot/manhole.py
+++ b/master/buildbot/manhole.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import os.path
import binascii, base64
from twisted.python import log
from twisted.application import service, strports
from twisted.cred import checkers, portal
from twisted.conch import manhole, telnet, manhole_ssh, checkers as conchc
from twisted.conch.insults import insults
--- a/master/buildbot/master.py
+++ b/master/buildbot/master.py
@@ -1,26 +1,39 @@
-# -*- test-case-name: buildbot.test.test_run -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import os
import signal
import time
import textwrap
from zope.interface import implements
from twisted.python import log, components
from twisted.python.failure import Failure
from twisted.internet import defer, reactor
from twisted.spread import pb
-from twisted.cred import portal, checkers
-from twisted.application import service, strports
+from twisted.application import service
from twisted.application.internet import TimerService
import buildbot
-# sibling imports
+import buildbot.pbmanager
from buildbot.util import now, safeTranslate, eventual
from buildbot.pbutil import NewCredPerspective
from buildbot.process.builder import Builder, IDLE
from buildbot.status.builder import Status, BuildSetStatus
from buildbot.changes.changes import Change
from buildbot.changes.manager import ChangeManager
from buildbot import interfaces, locks
from buildbot.process.properties import Properties
@@ -39,32 +52,33 @@ class BotMaster(service.MultiService):
"""This is the master-side service which manages remote buildbot slaves.
It provides them with BuildSlaves, and distributes file change
notification messages to them.
"""
debug = 0
reactor = reactor
- def __init__(self):
+ def __init__(self, master):
service.MultiService.__init__(self)
+ self.master = master
+
self.builders = {}
self.builderNames = []
# builders maps Builder names to instances of bb.p.builder.Builder,
# which is the master-side object that defines and controls a build.
# They are added by calling botmaster.addBuilder() from the startup
# code.
# self.slaves contains a ready BuildSlave instance for each
# potential buildslave, i.e. all the ones listed in the config file.
# If the slave is connected, self.slaves[slavename].slave will
# contain a RemoteReference to their Bot instance. If it is not
# connected, that attribute will hold None.
self.slaves = {} # maps slavename to BuildSlave
- self.statusClientService = None
self.watchers = {}
# self.locks holds the real Lock instances
self.locks = {}
# self.mergeRequests is the callable override for merging build
# requests
self.mergeRequests = None
@@ -73,16 +87,18 @@ class BotMaster(service.MultiService):
# traversal
self.prioritizeBuilders = None
self.loop = DelegateLoop(self._get_processors)
self.loop.setServiceParent(self)
self.shuttingDown = False
+ self.lastSlavePortnum = None
+
def setMasterName(self, name, incarnation):
self.master_name = name
self.master_incarnation = incarnation
def cleanShutdown(self):
if self.shuttingDown:
return
log.msg("Initiating clean shutdown")
@@ -197,16 +213,23 @@ class BotMaster(service.MultiService):
for sb in b.slaves:
if sb.state != IDLE:
d = defer.Deferred()
b.watchers['idle'].append(d)
return d
return defer.succeed(None)
def loadConfig_Slaves(self, new_slaves):
+ new_portnum = (self.lastSlavePortnum is not None
+ and self.lastSlavePortnum != self.master.slavePortnum)
+ if new_portnum:
+ # it turns out this is pretty hard..
+ raise ValueError("changing slavePortnum in reconfig is not supported")
+ self.lastSlavePortnum = self.master.slavePortnum
+
old_slaves = [c for c in list(self)
if interfaces.IBuildSlave.providedBy(c)]
# identify added/removed slaves. For each slave we construct a tuple
# of (name, password, class), and we consider the slave to be already
# present if the tuples match. (we include the class to make sure
# that BuildSlave(name,pw) is different than
# SubclassOfBuildSlave(name,pw) ). If the password or class has
@@ -225,39 +248,50 @@ class BotMaster(service.MultiService):
for t in old_t
if t not in new_t]
added = [new_t[t]
for t in new_t
if t not in old_t]
remaining_t = [t
for t in new_t
if t in old_t]
+
# removeSlave will hang up on the old bot
dl = []
for s in removed:
dl.append(self.removeSlave(s))
d = defer.DeferredList(dl, fireOnOneErrback=True)
- def _add(res):
+
+ def add_new(res):
for s in added:
self.addSlave(s)
+ d.addCallback(add_new)
+
+ def update_remaining(_):
for t in remaining_t:
old_t[t].update(new_t[t])
- d.addCallback(_add)
+ d.addCallback(update_remaining)
+
return d
def addSlave(self, s):
s.setServiceParent(self)
s.setBotmaster(self)
self.slaves[s.slavename] = s
+ s.pb_registration = self.master.pbmanager.register(
+ self.master.slavePortnum, s.slavename,
+ s.password, self.getPerspective)
def removeSlave(self, s):
- # TODO: technically, disownServiceParent could return a Deferred
- s.disownServiceParent()
- d = self.slaves[s.slavename].disconnect()
- del self.slaves[s.slavename]
+ d = s.disownServiceParent()
+ d.addCallback(lambda _ : s.pb_registration.unregister())
+ d.addCallback(lambda _ : self.slaves[s.slavename].disconnect())
+ def delslave(_):
+ del self.slaves[s.slavename]
+ d.addCallback(delslave)
return d
def slaveLost(self, bot):
for name, b in self.builders.items():
if bot.slavename in b.slavenames:
b.detached(bot)
def getBuildersForSlave(self, slavename):
@@ -305,69 +339,38 @@ class BotMaster(service.MultiService):
return defer.DeferredList(dl)
def shouldMergeRequests(self, builder, req1, req2):
"""Determine whether two BuildRequests should be merged for
the given builder.
"""
if self.mergeRequests is not None:
- return self.mergeRequests(builder, req1, req2)
+ if callable(self.mergeRequests):
+ return self.mergeRequests(builder, req1, req2)
+ elif self.mergeRequests == False:
+ # To save typing, this allows c['mergeRequests'] = False
+ return False
return req1.canBeMergedWith(req2)
def getPerspective(self, mind, slavename):
sl = self.slaves[slavename]
if not sl:
return None
# record when this connection attempt occurred
sl.recordConnectTime()
if sl.isConnected():
- # uh-oh, we've got a duplicate slave. The most likely
- # explanation is that the slave is behind a slow link, thinks we
- # went away, and has attempted to reconnect, so we've got two
- # "connections" from the same slave. The old may not be stale at this
- # point, if there are two slave proceses out there with the same name,
- # so instead of booting the old (which may be in the middle of a build),
- # we reject the new connection and ping the old slave.
- log.msg("duplicate slave %s; rejecting new slave and pinging old" % sl.slavename)
-
- # just in case we've got two identically-configured slaves,
- # report the IP addresses of both so someone can resolve the
- # squabble
- old_tport = sl.slave.broker.transport
- new_tport = mind.broker.transport
- log.msg("old slave was connected from", old_tport.getPeer())
- log.msg("new slave is from", new_tport.getPeer())
-
- # ping the old slave. If this kills it, then the new slave will connect
- # again and everyone will be happy.
- d = sl.slave.callRemote("print", "master got a duplicate connection; keeping this one")
-
- # now return a dummy avatar and kill the new connection in 5
- # seconds, thereby giving the ping a bit of time to kill the old
- # connection, if necessary
- def kill():
- log.msg("killing new slave on", new_tport.getPeer())
- new_tport.loseConnection()
- reactor.callLater(5, kill)
- class DummyAvatar(pb.Avatar):
- def attached(self, *args):
- pass
- def detached(self, *args):
- pass
- return DummyAvatar()
-
- return sl
-
- def shutdownSlaves(self):
- # TODO: make this into a bot method rather than a builder method
- for b in self.slaves.values():
- b.shutdownSlave()
+ # duplicate slave - send it to arbitration
+ arb = DuplicateSlaveArbitrator(sl)
+ return arb.getPerspective(mind, slavename)
+ else:
+ log.msg("slave '%s' attaching from %s" % (slavename, mind.broker.transport.getPeer()))
+ return sl
def stopService(self):
for b in self.builders.values():
b.builder_status.addPointEvent(["master", "shutdown"])
b.builder_status.saveYourself()
return service.MultiService.stopService(self)
def getLockByID(self, lockid):
@@ -379,16 +382,161 @@ class BotMaster(service.MultiService):
if not lockid in self.locks:
self.locks[lockid] = lockid.lockClass(lockid)
# if the master.cfg file has changed maxCount= on the lock, the next
# time a build is started, they'll get a new RealLock instance. Note
# that this requires that MasterLock and SlaveLock (marker) instances
# be hashable and that they should compare properly.
return self.locks[lockid]
+class DuplicateSlaveArbitrator(object):
+ """Utility class to arbitrate the situation when a new slave connects with
+ the name of an existing, connected slave"""
+ # There are several likely duplicate slave scenarios in practice:
+ #
+ # 1. two slaves are configured with the same username/password
+ #
+ # 2. the same slave process believes it is disconnected (due to a network
+ # hiccup), and is trying to reconnect
+ #
+ # For the first case, we want to prevent the two slaves from repeatedly
+ # superseding one another (which results in lots of failed builds), so we
+ # will prefer the old slave. However, for the second case we need to
+ # detect situations where the old slave is "gone". Sometimes "gone" means
+ # that the TCP/IP connection to it is in a long timeout period (10-20m,
+ # depending on the OS configuration), so this can take a while.
+
+ PING_TIMEOUT = 10
+ """Timeout for pinging the old slave. Set this to something quite long, as
+ a very busy slave (e.g., one sending a big log chunk) may take a while to
+ return a ping."""
+
+ def __init__(self, slave):
+ self.old_slave = slave
+ "L{buildbot.buildslave.AbstractSlaveBuilder} instance"
+
+ def getPerspective(self, mind, slavename):
+ self.new_slave_mind = mind
+
+ old_tport = self.old_slave.slave.broker.transport
+ new_tport = mind.broker.transport
+ log.msg("duplicate slave %s; delaying new slave (%s) and pinging old (%s)" %
+ (self.old_slave.slavename, new_tport.getPeer(), old_tport.getPeer()))
+
+ # delay the new slave until we decide what to do with it
+ self.new_slave_d = defer.Deferred()
+
+ # Ping the old slave. If this kills it, then we can allow the new
+ # slave to connect. If this does not kill it, then we disconnect
+ # the new slave.
+ self.ping_old_slave_done = False
+ self.old_slave_connected = True
+ self.ping_old_slave(new_tport.getPeer())
+
+ # Print a message on the new slave, if possible.
+ self.ping_new_slave_done = False
+ self.ping_new_slave()
+
+ return self.new_slave_d
+
+ def ping_new_slave(self):
+ d = self.new_slave_mind.callRemote("print",
+ "master already has a connection named '%s' - checking its liveness"
+ % self.old_slave.slavename)
+ def done(_):
+ # failure or success, doesn't matter
+ self.ping_new_slave_done = True
+ self.maybe_done()
+ d.addBoth(done)
+
+ def ping_old_slave(self, new_peer):
+ # set a timer on this ping, in case the network is bad. TODO: a timeout
+ # on the ping itself is not quite what we want. If there is other data
+ # flowing over the PB connection, then we should keep waiting. Bug #1703
+ def timeout():
+ self.ping_old_slave_timeout = None
+ self.ping_old_slave_timed_out = True
+ self.old_slave_connected = False
+ self.ping_old_slave_done = True
+ self.maybe_done()
+ self.ping_old_slave_timeout = reactor.callLater(self.PING_TIMEOUT, timeout)
+ self.ping_old_slave_timed_out = False
+
+ d = self.old_slave.slave.callRemote("print",
+ "master got a duplicate connection from %s; keeping this one" % new_peer)
+
+ def clear_timeout(r):
+ if self.ping_old_slave_timeout:
+ self.ping_old_slave_timeout.cancel()
+ self.ping_old_slave_timeout = None
+ return r
+ d.addBoth(clear_timeout)
+
+ def old_gone(f):
+ if self.ping_old_slave_timed_out:
+ return # ignore after timeout
+ f.trap(pb.PBConnectionLost)
+ log.msg(("connection lost while pinging old slave '%s' - " +
+ "keeping new slave") % self.old_slave.slavename)
+ self.old_slave_connected = False
+ d.addErrback(old_gone)
+
+ def other_err(f):
+ if self.ping_old_slave_timed_out:
+ return # ignore after timeout
+ log.msg("unexpected error while pinging old slave; disconnecting it")
+ log.err(f)
+ self.old_slave_connected = False
+ d.addErrback(other_err)
+
+ def done(_):
+ if self.ping_old_slave_timed_out:
+ return # ignore after timeout
+ self.ping_old_slave_done = True
+ self.maybe_done()
+ d.addCallback(done)
+
+ def maybe_done(self):
+ if not self.ping_new_slave_done or not self.ping_old_slave_done:
+ return
+
+ # both pings are done, so sort out the results
+ if self.old_slave_connected:
+ self.disconnect_new_slave()
+ else:
+ self.start_new_slave()
+
+ def start_new_slave(self, count=20):
+ if not self.new_slave_d:
+ return
+
+ # we need to wait until the old slave has actually disconnected, which
+ # can take a little while -- but don't wait forever!
+ if self.old_slave.isConnected():
+ if self.old_slave.slave:
+ self.old_slave.slave.broker.transport.loseConnection()
+ if count < 0:
+ log.msg("WEIRD: want to start new slave, but the old slave will not disconnect")
+ self.disconnect_new_slave()
+ else:
+ reactor.callLater(0.1, self.start_new_slave, count-1)
+ return
+
+ d = self.new_slave_d
+ self.new_slave_d = None
+ d.callback(self.old_slave)
+
+ def disconnect_new_slave(self):
+ if not self.new_slave_d:
+ return
+ d = self.new_slave_d
+ self.new_slave_d = None
+ log.msg("rejecting duplicate slave with exception")
+ d.errback(Failure(RuntimeError("rejecting duplicate slave")))
+
########################################
class DebugPerspective(NewCredPerspective):
def attached(self, mind):
return self
def detached(self, mind):
@@ -439,55 +587,16 @@ class DebugPerspective(NewCredPerspectiv
bot = s.f
for channel in bot.channels:
print " channel", channel
bot.p.msg(channel, "Ow, quit it")
def perspective_print(self, msg):
print "debug", msg
-class Dispatcher:
- implements(portal.IRealm)
-
- def __init__(self):
- self.names = {}
-
- def register(self, name, afactory):
- self.names[name] = afactory
- def unregister(self, name):
- del self.names[name]
-
- def requestAvatar(self, avatarID, mind, interface):
- assert interface == pb.IPerspective
- afactory = self.names.get(avatarID)
- if afactory:
- p = afactory.getPerspective()
- elif avatarID == "change":
- raise ValueError("no PBChangeSource installed")
- elif avatarID == "debug":
- p = DebugPerspective()
- p.master = self.master
- p.botmaster = self.botmaster
- elif avatarID == "statusClient":
- p = self.statusClientService.getPerspective()
- else:
- # it must be one of the buildslaves: no other names will make it
- # past the checker
- p = self.botmaster.getPerspective(mind, avatarID)
-
- if not p:
- raise ValueError("no perspective for '%s'" % avatarID)
-
- d = defer.maybeDeferred(p.attached, mind)
- def _avatarAttached(_, mind):
- return (pb.IPerspective, p, lambda: p.detached(mind))
- d.addCallback(_avatarAttached, mind)
- return d
-
-
########################################
class _Unset: pass # marker
class LogRotation:
'''holds log rotation parameters (for WebStatus)'''
def __init__(self):
self.rotateLength = 1 * 1000 * 1000
@@ -504,58 +613,52 @@ class BuildMaster(service.MultiService):
properties = Properties()
def __init__(self, basedir, configFileName="master.cfg", db_spec=None):
service.MultiService.__init__(self)
self.setName("buildmaster")
self.basedir = basedir
self.configFileName = configFileName
- # the dispatcher is the realm in which all inbound connections are
- # looked up: slave builders, change notifications, status clients, and
- # the debug port
- dispatcher = Dispatcher()
- dispatcher.master = self
- self.dispatcher = dispatcher
- self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
- # the checker starts with no user/passwd pairs: they are added later
- p = portal.Portal(dispatcher)
- p.registerChecker(self.checker)
- self.slaveFactory = pb.PBServerFactory(p)
- self.slaveFactory.unsafeTracebacks = True # let them see exceptions
+ self.pbmanager = buildbot.pbmanager.PBManager()
+ self.pbmanager.setServiceParent(self)
+ "L{buildbot.pbmanager.PBManager} instance managing connections for this master"
self.slavePortnum = None
self.slavePort = None
self.change_svc = ChangeManager()
self.change_svc.setServiceParent(self)
- self.dispatcher.changemaster = self.change_svc
try:
hostname = os.uname()[1] # only on unix
except AttributeError:
hostname = "?"
self.master_name = "%s:%s" % (hostname, os.path.abspath(self.basedir))
self.master_incarnation = "pid%d-boot%d" % (os.getpid(), time.time())
- self.botmaster = BotMaster()
+ self.botmaster = BotMaster(self)
self.botmaster.setName("botmaster")
self.botmaster.setMasterName(self.master_name, self.master_incarnation)
self.botmaster.setServiceParent(self)
- self.dispatcher.botmaster = self.botmaster
+
+ self.debugClientRegistration = None
self.status = Status(self.botmaster, self.basedir)
self.statusTargets = []
self.db = None
self.db_url = None
self.db_poll_interval = _Unset
if db_spec:
self.loadDatabase(db_spec)
+ # note that "read" here is taken in the past participal (i.e., "I read
+ # the config already") rather than the imperative ("you should read the
+ # config later")
self.readConfig = False
# create log_rotation object and set default parameters (used by WebStatus)
self.log_rotation = LogRotation()
def startService(self):
service.MultiService.startService(self)
if not self.readConfig:
@@ -681,18 +784,18 @@ class BuildMaster(service.MultiService):
if logMaxSize is not None and not \
isinstance(logMaxSize, int):
raise ValueError("logMaxSize needs to be None or int")
logMaxTailSize = config.get('logMaxTailSize')
if logMaxTailSize is not None and not \
isinstance(logMaxTailSize, int):
raise ValueError("logMaxTailSize needs to be None or int")
mergeRequests = config.get('mergeRequests')
- if mergeRequests is not None and not callable(mergeRequests):
- raise ValueError("mergeRequests must be a callable")
+ if mergeRequests not in (None, False) and not callable(mergeRequests):
+ raise ValueError("mergeRequests must be a callable or False")
prioritizeBuilders = config.get('prioritizeBuilders')
if prioritizeBuilders is not None and not callable(prioritizeBuilders):
raise ValueError("prioritizeBuilders must be callable")
changeHorizon = config.get("changeHorizon")
if changeHorizon is not None and not isinstance(changeHorizon, int):
raise ValueError("changeHorizon needs to be an int")
multiMaster = config.get("multiMaster", False)
@@ -894,31 +997,25 @@ class BuildMaster(service.MultiService):
if prioritizeBuilders is not None:
self.botmaster.prioritizeBuilders = prioritizeBuilders
self.buildCacheSize = buildCacheSize
self.changeCacheSize = changeCacheSize
self.eventHorizon = eventHorizon
self.logHorizon = logHorizon
self.buildHorizon = buildHorizon
+ self.slavePortnum = slavePortnum # TODO: move this to master.config.slavePortnum
# Set up the database
d.addCallback(lambda res:
self.loadConfig_Database(db_url, db_poll_interval))
- # self.slaves: Disconnect any that were attached and removed from the
- # list. Update self.checker with the new list of passwords, including
- # debug/change/status.
+ # set up slaves
d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
- # self.debugPassword
- if debugPassword:
- self.checker.addUser("debug", debugPassword)
- self.debugPassword = debugPassword
-
# self.manhole
if manhole != self.manhole:
# changing
if self.manhole:
# disownServiceParent may return a Deferred
d.addCallback(lambda res: self.manhole.disownServiceParent())
def _remove(res):
self.manhole = None
@@ -934,35 +1031,22 @@ class BuildMaster(service.MultiService):
# botmaster will handle startup/shutdown issues.
d.addCallback(lambda res: self.loadConfig_Builders(builders))
d.addCallback(lambda res: self.loadConfig_status(status))
# Schedulers are added after Builders in case they start right away
d.addCallback(lambda res:
self.scheduler_manager.updateSchedulers(schedulers))
+
# and Sources go after Schedulers for the same reason
d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
- # self.slavePort
- if self.slavePortnum != slavePortnum:
- if self.slavePort:
- def closeSlavePort(res):
- d1 = self.slavePort.disownServiceParent()
- self.slavePort = None
- return d1
- d.addCallback(closeSlavePort)
- if slavePortnum is not None:
- def openSlavePort(res):
- self.slavePort = strports.service(slavePortnum,
- self.slaveFactory)
- self.slavePort.setServiceParent(self)
- d.addCallback(openSlavePort)
- log.msg("BuildMaster listening on port %s" % slavePortnum)
- self.slavePortnum = slavePortnum
+ # debug client
+ d.addCallback(lambda res: self.loadConfig_DebugClient(debugPassword))
log.msg("configuration update started")
def _done(res):
self.readConfig = True
log.msg("configuration update complete")
d.addCallback(_done)
d.addCallback(lambda res: self.botmaster.triggerNewBuildCheck())
d.addErrback(log.err)
@@ -1019,22 +1103,16 @@ class BuildMaster(service.MultiService):
def loadConfig_Database(self, db_url, db_poll_interval):
self.db_url = db_url
self.db_poll_interval = db_poll_interval
db_spec = DBSpec.from_url(db_url, self.basedir)
self.loadDatabase(db_spec, db_poll_interval)
def loadConfig_Slaves(self, new_slaves):
- # set up the Checker with the names and passwords of all valid slaves
- self.checker.users = {} # violates abstraction, oh well
- for s in new_slaves:
- self.checker.addUser(s.slavename, s.password)
- self.checker.addUser("change", "changepw")
- # let the BotMaster take care of the rest
return self.botmaster.loadConfig_Slaves(new_slaves)
def loadConfig_Sources(self, sources):
if not sources:
log.msg("warning: no ChangeSources specified in c['change_source']")
# shut down any that were removed, start any that were added
deleted_sources = [s for s in self.change_svc if s not in sources]
added_sources = [s for s in sources if s not in self.change_svc]
@@ -1042,16 +1120,38 @@ class BuildMaster(service.MultiService):
(len(added_sources), len(deleted_sources)))
dl = [self.change_svc.removeSource(s) for s in deleted_sources]
def addNewOnes(res):
[self.change_svc.addSource(s) for s in added_sources]
d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0)
d.addCallback(addNewOnes)
return d
+ def loadConfig_DebugClient(self, debugPassword):
+ def makeDbgPerspective():
+ persp = DebugPerspective()
+ persp.master = self
+ persp.botmaster = self.botmaster
+ return persp
+
+ # unregister the old name..
+ if self.debugClientRegistration:
+ d = self.debugClientRegistration.unregister()
+ self.debugClientRegistration = None
+ else:
+ d = defer.succeed(None)
+
+ # and register the new one
+ def reg(_):
+ if debugPassword:
+ self.debugClientRegistration = self.pbmanager.register(
+ self.slavePortnum, "debug", debugPassword, makeDbgPerspective)
+ d.addCallback(reg)
+ return d
+
def allSchedulers(self):
return list(self.scheduler_manager)
def loadConfig_Builders(self, newBuilderData):
somethingChanged = False
newList = {}
newBuilderNames = []
allBuilders = self.botmaster.builders.copy()
new file mode 100644
--- /dev/null
+++ b/master/buildbot/pbmanager.py
@@ -0,0 +1,174 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from zope.interface import implements
+from twisted.spread import pb
+from twisted.python import failure, log
+from twisted.internet import defer
+from twisted.cred import portal, checkers, credentials, error
+from twisted.application import service, strports
+
+debug = False
+
+class PBManager(service.MultiService):
+ """
+ A centralized manager for PB ports and authentication on them.
+
+ Allows various pieces of code to request a (port, username) combo, along
+ with a password and a perspective factory.
+ """
+ def __init__(self):
+ service.MultiService.__init__(self)
+ self.dispatchers = {}
+
+ def register(self, portstr, username, password, pfactory):
+ """
+ Register a perspective factory PFACTORY to be executed when a PB
+ connection arrives on PORTSTR with USERNAME/PASSWORD. Returns a
+ Registration object which can be used to unregister later.
+ """
+ # do some basic normalization of portstrs
+ if type(portstr) == type(0) or ':' not in portstr:
+ portstr = "tcp:%s" % portstr
+
+ reg = Registration(self, portstr, username)
+
+ if portstr not in self.dispatchers:
+ disp = self.dispatchers[portstr] = Dispatcher(portstr)
+ disp.setServiceParent(self)
+ else:
+ disp = self.dispatchers[portstr]
+
+ disp.register(username, password, pfactory)
+
+ return reg
+
+ def _unregister(self, registration):
+ disp = self.dispatchers[registration.portstr]
+ disp.unregister(registration.username)
+ registration.username = None
+ if not disp.users:
+ disp = self.dispatchers[registration.portstr]
+ del self.dispatchers[registration.portstr]
+ return disp.disownServiceParent()
+ return defer.succeed(None)
+
+
+class Registration(object):
+ def __init__(self, pbmanager, portstr, username):
+ self.portstr = portstr
+ "portstr this registration is active on"
+ self.username = username
+ "username of this registration"
+
+ self.pbmanager = pbmanager
+
+ def unregister(self):
+ """
+ Unregister this registration, removing the username from the port, and
+ closing the port if there are no more users left. Returns a Deferred.
+ """
+ return self.pbmanager._unregister(self)
+
+
+class Dispatcher(service.MultiService):
+ implements(portal.IRealm, checkers.ICredentialsChecker)
+
+ credentialInterfaces = [ credentials.IUsernamePassword,
+ credentials.IUsernameHashedPassword ]
+
+ def __init__(self, portstr):
+ service.MultiService.__init__(self)
+ self.portstr = portstr
+ self.users = {}
+
+ # there's lots of stuff to set up for a PB connection!
+ self.portal = portal.Portal(self)
+ self.portal.registerChecker(self)
+ self.serverFactory = pb.PBServerFactory(self.portal)
+ self.serverFactory.unsafeTraceback = True
+ self.serverPort = strports.service(portstr, self.serverFactory)
+ self.serverPort.setServiceParent(self)
+
+ def getPort(self):
+ # helper method for testing
+ return self.serverPort.getHost().port
+
+ def startService(self):
+ return service.MultiService.startService(self)
+
+ def stopService(self):
+ return service.MultiService.stopService(self)
+
+ def register(self, username, password, pfactory):
+ if debug:
+ log.msg("registering username '%s' on pb port %s: %s"
+ % (username, self.portstr, pfactory))
+ if username in self.users:
+ raise KeyError, ("username '%s' is already registered on PB port %s"
+ % (username, self.portstr))
+ self.users[username] = (password, pfactory)
+
+ def unregister(self, username):
+ if debug:
+ log.msg("unregistering username '%s' on pb port %s"
+ % (username, self.portstr))
+ del self.users[username]
+
+ # IRealm
+
+ def requestAvatar(self, username, mind, interface):
+ assert interface == pb.IPerspective
+ if username not in self.users:
+ d = defer.succeed(None) # no perspective
+ else:
+ _, afactory = self.users.get(username)
+ d = defer.maybeDeferred(afactory, mind, username)
+
+ # check that we got a perspective
+ def check(persp):
+ if not persp:
+ raise ValueError("no perspective for '%s'" % username)
+ return persp
+ d.addCallback(check)
+
+ # call the perspective's attached(mind)
+ def call_attached(persp):
+ d = defer.maybeDeferred(persp.attached, mind)
+ d.addCallback(lambda _ : persp) # keep returning the perspective
+ return d
+ d.addCallback(call_attached)
+
+ # return the tuple requestAvatar is expected to return
+ def done(persp):
+ return (pb.IPerspective, persp, lambda: persp.detached(mind))
+ d.addCallback(done)
+
+ return d
+
+ # ICredentialsChecker
+
+ def requestAvatarId(self, creds):
+ if creds.username in self.users:
+ password, _ = self.users[creds.username]
+ d = defer.maybeDeferred(creds.checkPassword, password)
+ def check(matched):
+ if not matched:
+ return failure.Failure(error.UnauthorizedLogin())
+ return creds.username
+ d.addCallback(check)
+ return d
+ else:
+ return defer.fail(error.UnauthorizedLogin())
--- a/master/buildbot/pbutil.py
+++ b/master/buildbot/pbutil.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
"""Base classes handy for use with PB clients.
"""
from twisted.spread import pb
from twisted.spread.pb import PBClientFactory
from twisted.internet import protocol
--- a/master/buildbot/process/base.py
+++ b/master/buildbot/process/base.py
@@ -1,20 +1,34 @@
-# -*- test-case-name: buildbot.test.test_step -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import types
from zope.interface import implements
from twisted.python import log
from twisted.python.failure import Failure
from twisted.internet import reactor, defer, error
from buildbot import interfaces, locks
from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION, \
- RETRY, worst_status
+ RETRY, SKIPPED, worst_status
from buildbot.status.builder import Results
from buildbot.status.progress import BuildProgress
class Build:
"""I represent a single build by a single slave. Specialized Builders can
use subclasses of Build to hold status information unique to those build
processes.
@@ -405,17 +419,20 @@ class Build:
possible_overall_result = SUCCESS
else:
possible_overall_result = WARNINGS
if step.flunkOnWarnings:
possible_overall_result = FAILURE
elif result in (EXCEPTION, RETRY):
terminate = True
- self.result = worst_status(self.result, possible_overall_result)
+ # if we skipped this step, then don't adjust the build status
+ if result != SKIPPED:
+ self.result = worst_status(self.result, possible_overall_result)
+
return terminate
def lostRemote(self, remote=None):
# the slave went away. There are several possible reasons for this,
# and they aren't necessarily fatal. For now, kill the build, but
# TODO: see if we can resume the build when it reconnects.
log.msg("%s.lostRemote" % self)
self.remote = None
@@ -436,18 +453,17 @@ class Build:
if self.finished:
return
# TODO: include 'reason' in this point event
self.builder.builder_status.addPointEvent(['interrupt'])
self.stopped = True
if self.currentStep:
self.currentStep.interrupt(reason)
- self.result = FAILURE
- self.text.append("Interrupted")
+ self.result = EXCEPTION
if self._acquiringLock:
lock, access, d = self._acquiringLock
lock.stopWaitingUntilAvailable(self, access, d)
d.callback(None)
def allStepsDone(self):
if self.result == FAILURE:
@@ -459,17 +475,17 @@ class Build:
else:
text = ["build", "successful"]
text.extend(self.text)
return self.buildFinished(text, self.result)
def buildException(self, why):
log.msg("%s.buildException" % self)
log.err(why)
- self.buildFinished(["build", "exception"], FAILURE)
+ self.buildFinished(["build", "exception"], EXCEPTION)
def buildFinished(self, text, results):
"""This method must be called when the last Step has completed. It
marks the Build as complete and returns the Builder to the 'idle'
state.
It takes two arguments which describe the overall build status:
text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE.
--- a/master/buildbot/process/builder.py
+++ b/master/buildbot/process/builder.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import random, weakref
from zope.interface import implements
from twisted.python import log
from twisted.python.failure import Failure
from twisted.spread import pb
from twisted.application import service, internet
from twisted.internet import defer
@@ -386,16 +401,17 @@ class Builder(pb.Referenceable, service.
self.nextBuild = setup.get('nextBuild')
if self.nextBuild is not None and not callable(self.nextBuild):
raise ValueError("nextBuild must be callable")
self.buildHorizon = setup.get('buildHorizon')
self.logHorizon = setup.get('logHorizon')
self.eventHorizon = setup.get('eventHorizon')
self.mergeRequests = setup.get('mergeRequests', True)
self.properties = setup.get('properties', {})
+ self.category = setup.get('category', None)
# build/wannabuild slots: Build objects move along this sequence
self.building = []
# old_building holds active builds that were stolen from a predecessor
self.old_building = weakref.WeakKeyDictionary()
# buildslaves which have connected but which are not yet available.
# These are always in the ATTACHING state.
@@ -451,16 +467,19 @@ class Builder(pb.Referenceable, service.
if setup.get('nextBuild') != self.nextBuild:
diffs.append('nextBuild changed from %s to %s' % (self.nextBuild, setup.get('nextBuild')))
if setup['buildHorizon'] != self.buildHorizon:
diffs.append('buildHorizon changed from %s to %s' % (self.buildHorizon, setup['buildHorizon']))
if setup['logHorizon'] != self.logHorizon:
diffs.append('logHorizon changed from %s to %s' % (self.logHorizon, setup['logHorizon']))
if setup['eventHorizon'] != self.eventHorizon:
diffs.append('eventHorizon changed from %s to %s' % (self.eventHorizon, setup['eventHorizon']))
+ if setup['category'] != self.category:
+ diffs.append('category changed from %r to %r' % (self.category, setup['category']))
+
return diffs
def __repr__(self):
return "<Builder '%r' at %d>" % (self.name, id(self))
def triggerNewBuildCheck(self):
self.botmaster.triggerNewBuildCheck()
@@ -610,16 +629,20 @@ class Builder(pb.Referenceable, service.
it wants."""
log.msg("consumeTheSoulOfYourPredecessor: %s feeding upon %s" %
(self, old))
# all pending builds are stored in the DB, so we don't have to do
# anything to claim them. The old builder will be stopService'd,
# which should make sure they don't start any new work
+ # this is kind of silly, but the builder status doesn't get updated
+ # when the config changes, yet it stores the category. So:
+ self.builder_status.category = self.category
+
# old.building (i.e. builds which are still running) is not migrated
# directly: it keeps track of builds which were in progress in the
# old Builder. When those builds finish, the old Builder will be
# notified, not us. However, since the old SlaveBuilder will point to
# us, it is our maybeStartBuild() that will be triggered.
if old.building:
self.builder_status.setBigState("building")
# however, we do grab a weakref to the active builds, so that our
@@ -657,24 +680,28 @@ class Builder(pb.Referenceable, service.
# from the old Builder), some of which will be new. The new ones
# will be re-attached.
# Therefore, we don't need to do anything about old.attaching_slaves
return # all done
def reclaimAllBuilds(self):
- now = util.now()
- brids = set()
- for b in self.building:
- brids.update([br.id for br in b.requests])
- for b in self.old_building:
- brids.update([br.id for br in b.requests])
- self.db.claim_buildrequests(now, self.master_name,
- self.master_incarnation, brids)
+ try:
+ now = util.now()
+ brids = set()
+ for b in self.building:
+ brids.update([br.id for br in b.requests])
+ for b in self.old_building:
+ brids.update([br.id for br in b.requests])
+ self.db.claim_buildrequests(now, self.master_name,
+ self.master_incarnation, brids)
+ except:
+ log.msg("Error in reclaimAllBuilds")
+ log.err()
def getBuild(self, number):
for b in self.building:
if b.build_status and b.build_status.number == number:
return b
for b in self.old_building.keys():
if b.build_status and b.build_status.number == number:
return b
--- a/master/buildbot/process/buildstep.py
+++ b/master/buildbot/process/buildstep.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_steps -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import re
from zope.interface import implements
from twisted.internet import reactor, defer, error
from twisted.protocols import basic
from twisted.spread import pb
from twisted.python import log
@@ -746,17 +760,17 @@ class BuildStep:
# all locks are available, claim them all
for lock, access in self.locks:
lock.claim(self, access)
self.step_status.setWaitingForLocks(False)
return defer.succeed(None)
def _startStep_2(self, res):
if self.stopped:
- self.finished(FAILURE)
+ self.finished(EXCEPTION)
return
if self.progress:
self.progress.start()
try:
skip = None
if isinstance(self.doStepIf, bool):
@@ -766,19 +780,21 @@ class BuildStep:
skip = SKIPPED
if skip is None:
skip = self.start()
if skip == SKIPPED:
self.step_status.setText(self.describe(True) + ['skipped'])
self.step_status.setSkipped(True)
- # this return value from self.start is a shortcut
- # to finishing the step immediately
- reactor.callLater(0, self.finished, SKIPPED)
+ # this return value from self.start is a shortcut to finishing
+ # the step immediately; we skip calling finished() as
+ # subclasses may have overridden that an expect it to be called
+ # after start() (bug #837)
+ reactor.callLater(0, self._finishFinished, SKIPPED)
except:
log.msg("BuildStep.startStep exception in .start")
self.failed(Failure())
def start(self):
"""Begin the step. Override this method and add code to do local
processing, fire off remote commands, etc.
@@ -857,16 +873,29 @@ class BuildStep:
for lock, access in self.locks:
if lock.isOwner(self, access):
lock.release(self, access)
else:
# This should only happen if we've been interrupted
assert self.stopped
def finished(self, results):
+ if self.stopped:
+ # We handle this specially because we don't care about
+ # the return code of an interrupted command; we know
+ # that this should just be exception due to interrupt
+ results = EXCEPTION
+ self.step_status.setText(self.describe(True) +
+ ["interrupted"])
+ self.step_status.setText2(["interrupted"])
+ self._finishFinished(results)
+
+ def _finishFinished(self, results):
+ # internal function to indicate that this step is done; this is separated
+ # from finished() so that subclasses can override finished()
if self.progress:
self.progress.finish()
self.step_status.stepFinished(results)
self.releaseLocks()
self.deferred.callback(results)
def failed(self, why):
# if isinstance(why, pb.CopiedFailure): # a remote exception might
@@ -1108,18 +1137,18 @@ class LoggingBuildStep(BuildStep):
if self.cmd:
d = self.cmd.interrupt(reason)
return d
def checkDisconnect(self, f):
f.trap(error.ConnectionLost)
self.step_status.setText(self.describe(True) +
- ["failed", "slave", "lost"])
- self.step_status.setText2(["failed", "slave", "lost"])
+ ["exception", "slave", "lost"])
+ self.step_status.setText2(["exception", "slave", "lost"])
return self.finished(RETRY)
# to refine the status output, override one or more of the following
# methods. Change as little as possible: start with the first ones on
# this list and only proceed further if you have to
#
# createSummary: add additional Logfiles with summarized results
# evaluateCommand: decides whether the step was successful or not
@@ -1165,16 +1194,18 @@ class LoggingBuildStep(BuildStep):
return FAILURE
return SUCCESS
def getText(self, cmd, results):
if results == SUCCESS:
return self.describe(True)
elif results == WARNINGS:
return self.describe(True) + ["warnings"]
+ elif results == EXCEPTION:
+ return self.describe(True) + ["exception"]
else:
return self.describe(True) + ["failed"]
def getText2(self, cmd, results):
"""We have decided to add a short note about ourselves to the overall
build description, probably because something went wrong. Return a
short list of short strings. If your subclass counts test failures or
warnings of some sort, this is a good place to announce the count."""
--- a/master/buildbot/process/factory.py
+++ b/master/buildbot/process/factory.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_step -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot import util
from buildbot.process.base import Build
from buildbot.process.buildstep import BuildStep
from buildbot.steps.source import CVS, SVN
from buildbot.steps.shell import Configure, Compile, Test, PerlModuleTest
# deprecated, use BuildFactory.addStep
--- a/master/buildbot/process/mtrlogobserver.py
+++ b/master/buildbot/process/mtrlogobserver.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import sys
import re
import exceptions
from twisted.python import log
from twisted.internet import defer
from twisted.enterprise import adbapi
from buildbot.process.buildstep import LogLineObserver
from buildbot.steps.shell import Test
--- a/master/buildbot/process/process_twisted.py
+++ b/master/buildbot/process/process_twisted.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# Build classes specific to the Twisted codebase
from buildbot.process.base import Build
from buildbot.process.factory import BuildFactory
from buildbot.steps import shell
from buildbot.steps.python_twisted import HLint, ProcessDocs, BuildDebs, \
Trial, RemovePYCs
--- a/master/buildbot/process/properties.py
+++ b/master/buildbot/process/properties.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import re
import weakref
from buildbot import util
class Properties(util.ComparableMixin):
"""
I represent a set of properties that can be interpolated into various
strings in buildsteps.
@@ -119,77 +134,105 @@ class PropertyMap:
including the rendering of None as ''.
"""
colon_minus_re = re.compile(r"(.*):-(.*)")
colon_tilde_re = re.compile(r"(.*):~(.*)")
colon_plus_re = re.compile(r"(.*):\+(.*)")
def __init__(self, properties):
# use weakref here to avoid a reference loop
self.properties = weakref.ref(properties)
+ self.temp_vals = {}
def __getitem__(self, key):
properties = self.properties()
assert properties is not None
def colon_minus(mo):
# %(prop:-repl)s
# if prop exists, use it; otherwise, use repl
prop, repl = mo.group(1,2)
- if properties.has_key(prop):
+ if prop in self.temp_vals:
+ return self.temp_vals[prop]
+ elif properties.has_key(prop):
return properties[prop]
else:
return repl
def colon_tilde(mo):
# %(prop:~repl)s
# if prop exists and is true (nonempty), use it; otherwise, use repl
prop, repl = mo.group(1,2)
- if properties.has_key(prop) and properties[prop]:
+ if prop in self.temp_vals and self.temp_vals[prop]:
+ return self.temp_vals[prop]
+ elif properties.has_key(prop) and properties[prop]:
return properties[prop]
else:
return repl
def colon_plus(mo):
# %(prop:+repl)s
# if prop exists, use repl; otherwise, an empty string
prop, repl = mo.group(1,2)
- if properties.has_key(prop):
+ if properties.has_key(prop) or prop in self.temp_vals:
return repl
else:
return ''
for regexp, fn in [
( self.colon_minus_re, colon_minus ),
( self.colon_tilde_re, colon_tilde ),
( self.colon_plus_re, colon_plus ),
]:
mo = regexp.match(key)
if mo:
rv = fn(mo)
break
else:
- rv = properties[key]
+ # If explicitly passed as a kwarg, use that,
+ # otherwise, use the property value.
+ if key in self.temp_vals:
+ rv = self.temp_vals[key]
+ else:
+ rv = properties[key]
# translate 'None' to an empty string
if rv is None: rv = ''
return rv
+ def add_temporary_value(self, key, val):
+ 'Add a temporary value (to support keyword arguments to WithProperties)'
+ self.temp_vals[key] = val
+
+ def clear_temporary_values(self):
+ self.temp_vals = {}
+
class WithProperties(util.ComparableMixin):
"""
This is a marker class, used fairly widely to indicate that we
want to interpolate build properties.
"""
compare_attrs = ('fmtstring', 'args')
- def __init__(self, fmtstring, *args):
+ def __init__(self, fmtstring, *args, **lambda_subs):
self.fmtstring = fmtstring
self.args = args
+ if not self.args:
+ self.lambda_subs = lambda_subs
+ for key, val in self.lambda_subs.iteritems():
+ if not callable(val):
+ raise ValueError('Value for lambda substitution "%s" must be callable.' % key)
+ elif lambda_subs:
+ raise ValueError('WithProperties takes either positional or keyword substitutions, not both.')
def render(self, pmap):
if self.args:
strings = []
for name in self.args:
strings.append(pmap[name])
s = self.fmtstring % tuple(strings)
else:
+ properties = pmap.properties()
+ for k,v in self.lambda_subs.iteritems():
+ pmap.add_temporary_value(k, v(properties))
s = self.fmtstring % pmap
+ pmap.clear_temporary_values()
return s
old mode 100755
new mode 100644
--- a/master/buildbot/process/subunitlogobserver.py
+++ b/master/buildbot/process/subunitlogobserver.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_buildstep -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from unittest import TestResult
from buildbot.process import buildstep
from StringIO import StringIO
class SubunitLogObserver(buildstep.LogLineObserver, TestResult):
"""Observe a log that may contain subunit output.
--- a/master/buildbot/scheduler.py
+++ b/master/buildbot/scheduler.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.schedulers.basic import Scheduler, AnyBranchScheduler, Dependent
from buildbot.schedulers.timed import Periodic, Nightly
from buildbot.schedulers.triggerable import Triggerable
from buildbot.schedulers.trysched import Try_Jobdir, Try_Userpass
_hush_pyflakes = [Scheduler, AnyBranchScheduler, Dependent,
Periodic, Nightly, Triggerable, Try_Jobdir, Try_Userpass]
--- a/master/buildbot/schedulers/base.py
+++ b/master/buildbot/schedulers/base.py
@@ -1,44 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
from zope.interface import implements
from twisted.application import service
from buildbot import interfaces
from buildbot.process.properties import Properties
from buildbot.util import ComparableMixin, NotABranch
from buildbot.schedulers import filter
--- a/master/buildbot/schedulers/basic.py
+++ b/master/buildbot/schedulers/basic.py
@@ -1,44 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
import time
from buildbot import interfaces, util
from buildbot.util import collections, NotABranch
from buildbot.sourcestamp import SourceStamp
from buildbot.status.builder import SUCCESS, WARNINGS
from buildbot.schedulers import base
@@ -136,33 +114,44 @@ class Scheduler(base.BaseScheduler, base
self.stableAt = most_recent + self.treeStableTimer
if self.stableAt > now:
# Wake up one second late, to avoid waking up too early and
# looping a lot.
return self.stableAt + 1.0
# ok, do a build
self.stableAt = None
- self._add_build_and_remove_changes(t, all_changes)
+ self._add_build_and_remove_changes(t, important, unimportant)
return None
- def _add_build_and_remove_changes(self, t, all_changes):
+ def _add_build_and_remove_changes(self, t, important, unimportant):
+ # the changes are segregated into important and unimportant
+ # changes, but we need it ordered earliest to latest, based
+ # on change number, since the SourceStamp will be created
+ # based on the final change.
+ all_changes = sorted(important + unimportant, key=lambda c : c.number)
+
db = self.parent.db
if self.treeStableTimer is None:
- # each Change gets a separate build
- for c in all_changes:
+ # each *important* Change gets a separate build. Unimportant
+ # builds get ignored.
+ for c in sorted(important, key=lambda c : c.number):
ss = SourceStamp(changes=[c])
ssid = db.get_sourcestampid(ss, t)
self.create_buildset(ssid, "scheduler", t)
else:
+ # if we had a treeStableTimer, then trigger a build for the
+ # whole pile - important or not. There's at least one important
+ # change in the list, or we wouldn't have gotten here.
ss = SourceStamp(changes=all_changes)
ssid = db.get_sourcestampid(ss, t)
self.create_buildset(ssid, "scheduler", t)
- # and finally retire the changes from scheduler_changes
+ # and finally retire all of the changes from scheduler_changes, regardless
+ # of importance level
changeids = [c.number for c in all_changes]
db.scheduler_retire_changes(self.schedulerid, changeids, t)
# the waterfall needs to know the next time we plan to trigger a build
def getPendingBuildTimes(self):
if self.stableAt and self.stableAt > util.now():
return [ self.stableAt ]
return []
--- a/master/buildbot/schedulers/filter.py
+++ b/master/buildbot/schedulers/filter.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import re, types
from buildbot.util import ComparableMixin, NotABranch
class ChangeFilter(ComparableMixin):
# TODO: filter_fn will always be different. Does that mean that we always
# reconfigure schedulers? Is that a problem?
--- a/master/buildbot/schedulers/manager.py
+++ b/master/buildbot/schedulers/manager.py
@@ -1,44 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
from twisted.internet import defer
from twisted.python import log
from buildbot.util import loop
from buildbot.util import collections
from buildbot.util.eventual import eventually
class SchedulerManager(loop.MultiServiceLoop):
--- a/master/buildbot/schedulers/timed.py
+++ b/master/buildbot/schedulers/timed.py
@@ -1,44 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
import time
from twisted.internet import defer
from twisted.python import log
from buildbot.sourcestamp import SourceStamp
from buildbot.schedulers import base
class TimedBuildMixin:
--- a/master/buildbot/schedulers/triggerable.py
+++ b/master/buildbot/schedulers/triggerable.py
@@ -1,44 +1,22 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
from twisted.internet import reactor, defer
from buildbot.schedulers import base
from buildbot.process.properties import Properties
class Triggerable(base.BaseScheduler):
"""This scheduler doesn't do anything until it is triggered by a Trigger
step in a factory. In general, that step will not complete until all of
--- a/master/buildbot/schedulers/trysched.py
+++ b/master/buildbot/schedulers/trysched.py
@@ -1,53 +1,28 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla-specific Buildbot steps.
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
-# The Initial Developer of the Original Code is
-# Mozilla Foundation.
-# Portions created by the Initial Developer are Copyright (C) 2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-# Brian Warner <warner@lothar.com>
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
-# ***** END LICENSE BLOCK *****
+# Copyright Buildbot Team Members
import os.path
-from zope.interface import implements
-from twisted.application import strports
+from twisted.internet import defer
from twisted.python import log, runtime
from twisted.protocols import basic
-from twisted.cred import portal, checkers
-from twisted.spread import pb
from buildbot import pbutil
from buildbot.sourcestamp import SourceStamp
from buildbot.changes.maildir import MaildirService
from buildbot.process.properties import Properties
from buildbot.schedulers import base
from buildbot.status.builder import BuildSetStatus
@@ -182,46 +157,46 @@ class Try_Jobdir(TryBase):
return d
def _try(self, t, ss, builderNames, reason):
db = self.parent.db
ssid = db.get_sourcestampid(ss, t)
bsid = self.create_buildset(ssid, reason, t, builderNames=builderNames)
return bsid
+
class Try_Userpass(TryBase):
compare_attrs = ( 'name', 'builderNames', 'port', 'userpass', 'properties' )
- implements(portal.IRealm)
def __init__(self, name, builderNames, port, userpass,
properties={}):
base.BaseScheduler.__init__(self, name, builderNames, properties)
- if type(port) is int:
- port = "tcp:%d" % port
self.port = port
self.userpass = userpass
- c = checkers.InMemoryUsernamePasswordDatabaseDontUse()
- for user,passwd in self.userpass:
- c.addUser(user, passwd)
+ self.properties = properties
- p = portal.Portal(self)
- p.registerChecker(c)
- f = pb.PBServerFactory(p)
- s = strports.service(port, f)
- s.setServiceParent(self)
+ def startService(self):
+ TryBase.startService(self)
+ master = self.parent.parent
- def getPort(self):
- # utility method for tests: figure out which TCP port we just opened.
- return self.services[0]._port.getHost().port
+ # register each user/passwd with the pbmanager
+ def factory(mind, username):
+ return Try_Userpass_Perspective(self, username)
+ self.registrations = []
+ for user, passwd in self.userpass:
+ self.registrations.append(
+ master.pbmanager.register(self.port, user, passwd, factory))
- def requestAvatar(self, avatarID, mind, interface):
- log.msg("%s got connection from user %s" % (self, avatarID))
- assert interface == pb.IPerspective
- p = Try_Userpass_Perspective(self, avatarID)
- return (pb.IPerspective, p, lambda: None)
+ def stopService(self):
+ d = defer.maybeDeferred(TryBase.stopService, self)
+ def unreg(_):
+ return defer.gatherResults(
+ [ reg.unregister() for reg in self.registrations ])
+ d.addCallback(unreg)
+
class Try_Userpass_Perspective(pbutil.NewCredPerspective):
def __init__(self, parent, username):
self.parent = parent
self.username = username
def perspective_try(self, branch, revision, patch, repository, project,
builderNames, properties={}, ):
--- a/master/buildbot/scripts/checkconfig.py
+++ b/master/buildbot/scripts/checkconfig.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import sys
import os
from shutil import copy, rmtree
from tempfile import mkdtemp
from os.path import isfile
from buildbot import master
@@ -25,14 +40,14 @@ class ConfigLoader(master.BuildMaster):
try:
os.chdir(tempdir)
# Add the temp directory to the library path so local modules work
sys.path.append(tempdir)
configFile = open(configFileName, "r")
self.loadConfig(configFile, check_synchronously_only=True)
except:
+ # clean up before passing on the exception
os.chdir(dir)
- configFile.close()
rmtree(tempdir)
raise
os.chdir(dir)
rmtree(tempdir)
--- a/master/buildbot/scripts/logwatcher.py
+++ b/master/buildbot/scripts/logwatcher.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import os
from twisted.python.failure import Failure
from twisted.internet import defer, reactor, protocol, error
from twisted.protocols.basic import LineOnlyReceiver
class FakeTransport:
disconnecting = False
--- a/master/buildbot/scripts/reconfig.py
+++ b/master/buildbot/scripts/reconfig.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import os, signal, platform
from twisted.internet import reactor
from buildbot.scripts.logwatcher import LogWatcher, BuildmasterTimeoutError, \
ReconfigError
class Reconfigurator:
--- a/master/buildbot/scripts/runner.py
+++ b/master/buildbot/scripts/runner.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_runner -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# N.B.: don't import anything that might pull in a reactor yet. Some of our
# subcommands want to load modules that need the gtk reactor.
#
# Also don't forget to mirror your changes on command-line options in manual
# pages and texinfo documentation.
import os, sys, stat, re, time
@@ -453,17 +467,17 @@ def upgradeMaster(config):
})
m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"),
util.sibpath(__file__, "sample.cfg"),
overwrite=True)
# if index.html exists, use it to override the root page tempalte
m.move_if_present(os.path.join(basedir, "public_html/index.html"),
os.path.join(basedir, "templates/root.html"))
- from buildbot.db import connector, dbspec
+ from buildbot.db import dbspec
spec = dbspec.DBSpec.from_url(config["db"], basedir)
# TODO: check that TAC file specifies the right spec
# upgrade the db
from buildbot.db.schema import manager
sm = manager.DBSchemaManager(spec, basedir)
sm.upgrade()
@@ -629,17 +643,17 @@ def stop(config, signame="TERM", wait=Fa
time.sleep(1)
if not quiet:
print "never saw process go away"
def restart(config):
basedir = config['basedir']
quiet = config['quiet']
- if not isBuildmasterDir(config['basedir']):
+ if not isBuildmasterDir(basedir):
print "not a buildmaster directory"
sys.exit(1)
from buildbot.scripts.startup import start
try:
stop(config, wait=True)
except BuildbotNotRunningError:
pass
@@ -766,34 +780,38 @@ def statusgui(config):
class SendChangeOptions(OptionsWithOptionsFile):
def __init__(self):
OptionsWithOptionsFile.__init__(self)
self['properties'] = {}
optParameters = [
("master", "m", None,
"Location of the buildmaster's PBListener (host:port)"),
- ("username", "u", None, "Username performing the commit"),
- ("repository", "R", None, "Repository specifier"),
- ("project", "P", None, "Project specifier"),
+ # deprecated in 0.8.3; remove in 0.8.5 (bug #1711)
+ ("username", "u", None, "deprecated name for --who"),
+ ("auth", "a", None, "Authentication token - username:password, or prompt for password"),
+ ("who", "W", None, "Author of the commit"),
+ ("repository", "R", '', "Repository specifier"),
+ ("project", "P", '', "Project specifier"),
("branch", "b", None, "Branch specifier"),
("category", "C", None, "Category of repository"),
("revision", "r", None, "Revision specifier"),
("revision_file", None, None, "Filename containing revision spec"),
("property", "p", None,
"A property for the change, in the format: name:value"),
("comments", "c", None, "log message"),
("logfile", "F", None,
"Read the log messages from this file (- for stdin)"),
("when", "w", None, "timestamp to use as the change time"),
- ("revlink", "l", None, "Revision link (revlink)"),
+ ("revlink", "l", '', "Revision link (revlink)"),
]
buildbotOptions = [
[ 'master', 'master' ],
+ [ 'who', 'who' ],
[ 'username', 'username' ],
[ 'branch', 'branch' ],
[ 'category', 'category' ],
]
def getSynopsis(self):
return "Usage: buildbot sendchange [options] filenames.."
def parseArgs(self, *args):
@@ -803,17 +821,21 @@ class SendChangeOptions(OptionsWithOptio
self['properties'][name] = value
def sendchange(config, runReactor=False):
"""Send a single change to the buildmaster's PBChangeSource. The
connection will be drpoped as soon as the Change has been sent."""
from buildbot.clients.sendchange import Sender
- user = config.get('username')
+ who = config.get('who')
+ if not who and config.get('username'):
+ print "NOTE: --username/-u is deprecated: use --who/-W'"
+ who = config.get('username')
+ auth = config.get('auth')
master = config.get('master')
branch = config.get('branch')
category = config.get('category')
revision = config.get('revision')
properties = config.get('properties', {})
repository = config.get('repository', '')
project = config.get('project', '')
revlink = config.get('revlink', '')
@@ -831,21 +853,30 @@ def sendchange(config, runReactor=False)
else:
f = open(config['logfile'], "rt")
comments = f.read()
if comments is None:
comments = ""
files = config.get('files', [])
- assert user, "you must provide a username"
+ # fix up the auth with a password if none was given
+ if not auth:
+ auth = 'change:changepw'
+ if ':' not in auth:
+ import getpass
+ pw = getpass.getpass("Enter password for '%s': " % auth)
+ auth = "%s:%s" % (auth, pw)
+ auth = auth.split(':', 1)
+
+ assert who, "you must provide a committer (--who)"
assert master, "you must provide the master location"
- s = Sender(master, user)
- d = s.send(branch, revision, comments, files, category=category, when=when,
+ s = Sender(master, auth)
+ d = s.send(branch, revision, comments, files, who=who, category=category, when=when,
properties=properties, repository=repository, project=project,
revlink=revlink)
if runReactor:
status = [True]
def failed(res):
status[0] = False
s.printFailure(res)
d.addCallbacks(s.printSuccess, failed)
@@ -982,16 +1013,17 @@ class TryServerOptions(OptionsWithOption
]
def getSynopsis(self):
return "Usage: buildbot tryserver [options]"
def doTryServer(config):
try:
from hashlib import md5
+ assert md5
except ImportError:
# For Python 2.4 compatibility
import md5
jobdir = os.path.expanduser(config["jobdir"])
job = sys.stdin.read()
# now do a 'safecat'-style write to jobdir/tmp, then move atomically to
# jobdir/new . Rather than come up with a unique name randomly, I'm just
# going to MD5 the contents and prepend a timestamp.
--- a/master/buildbot/scripts/sample.cfg
+++ b/master/buildbot/scripts/sample.cfg
@@ -1,205 +1,116 @@
# -*- python -*-
# ex: set syntax=python:
# This is a sample buildmaster config file. It must be installed as
-# 'master.cfg' in your buildmaster's base directory (although the filename
-# can be changed with the --basedir option to 'mktap buildbot master').
-
-# It has one job: define a dictionary named BuildmasterConfig. This
-# dictionary has a variety of keys to control different aspects of the
-# buildmaster. They are documented in docs/config.xhtml .
-
+# 'master.cfg' in your buildmaster's base directory.
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
-####### DB URL
-
-# This specifies what database buildbot uses to store change and scheduler
-# state
-c['db_url'] = "sqlite:///state.sqlite"
-
####### BUILDSLAVES
-# the 'slaves' list defines the set of allowable buildslaves. Each element is
-# a BuildSlave object, which is created with bot-name, bot-password. These
-# correspond to values given to the buildslave's mktap invocation.
+# The 'slaves' list defines the set of recognized buildslaves. Each element is
+# a BuildSlave object, specifying a username and password. The same username and
+# password must be configured on the slave.
from buildbot.buildslave import BuildSlave
-c['slaves'] = [BuildSlave("bot1name", "bot1passwd")]
+c['slaves'] = [BuildSlave("example-slave", "pass")]
-# to limit to two concurrent builds on a slave, use
-# c['slaves'] = [BuildSlave("bot1name", "bot1passwd", max_builds=2)]
-
-
-# 'slavePortnum' defines the TCP port to listen on. This must match the value
-# configured into the buildslaves (with their --master option)
-
+# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
+# This must match the value configured into the buildslaves (with their
+# --master option)
c['slavePortnum'] = 9989
####### CHANGESOURCES
# the 'change_source' setting tells the buildmaster how it should find out
-# about source code changes. Any class which implements IChangeSource can be
-# put here: there are several in buildbot/changes/*.py to choose from.
-
-from buildbot.changes.pb import PBChangeSource
-c['change_source'] = PBChangeSource()
-
-# For example, if you had CVSToys installed on your repository, and your
-# CVSROOT/freshcfg file had an entry like this:
-#pb = ConfigurationSet([
-# (None, None, None, PBService(userpass=('foo', 'bar'), port=4519)),
-# ])
-
-# then you could use the following buildmaster Change Source to subscribe to
-# the FreshCVS daemon and be notified on every commit:
-#
-#from buildbot.changes.freshcvs import FreshCVSSource
-#fc_source = FreshCVSSource("cvs.example.com", 4519, "foo", "bar")
-#c['change_source'] = fc_source
+# about source code changes. Here we point to the buildbot clone of pyflakes.
-# or, use a PBChangeSource, and then have your repository's commit script run
-# 'buildbot sendchange', or use contrib/svn_buildbot.py, or
-# contrib/arch_buildbot.py :
-#
-#from buildbot.changes.pb import PBChangeSource
-#c['change_source'] = PBChangeSource()
-
-# If you wat to use SVNPoller, it might look something like
-# # Where to get source code changes
-# from buildbot.changes.svnpoller import SVNPoller
-# source_code_svn_url='https://svn.myproject.org/bluejay/trunk'
-# svn_poller = SVNPoller(
-# svnurl=source_code_svn_url,
-# pollinterval=60*60, # seconds
-# histmax=10,
-# svnbin='/usr/bin/svn',
-## )
-# c['change_source'] = [ svn_poller ]
+from buildbot.changes.gitpoller import GitPoller
+c['change_source'] = GitPoller(
+ 'git://github.com/buildbot/pyflakes.git',
+ branch='master', pollinterval=1200)
####### SCHEDULERS
-## configure the Schedulers
+# Configure the Schedulers, which decide how to react to incoming changes. In this
+# case, just kick off a 'runtests' build
from buildbot.scheduler import Scheduler
c['schedulers'] = []
c['schedulers'].append(Scheduler(name="all", branch=None,
- treeStableTimer=2*60,
- builderNames=["buildbot-full"]))
-
+ treeStableTimer=None,
+ builderNames=["runtests"]))
####### BUILDERS
-# the 'builders' list defines the Builders. Each one is configured with a
-# dictionary, using the following keys:
-# name (required): the name used to describe this builder
-# slavename or slavenames (required): which slave(s) to use (must appear in c['slaves'])
-# factory (required): a BuildFactory to define how the build is run
-# builddir (optional): which subdirectory to run the builder in
-
-# buildbot/process/factory.py provides several BuildFactory classes you can
-# start with, which implement build processes for common targets (GNU
-# autoconf projects, CPAN perl modules, etc). The factory.BuildFactory is the
-# base class, and is configured with a series of BuildSteps. When the build
-# is run, the appropriate buildslave is told to execute each Step in turn.
+# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
+# what steps, and which slaves can execute them. Note that any particular build will
+# only take place on one slave.
-# the first BuildStep is typically responsible for obtaining a copy of the
-# sources. There are source-obtaining Steps in buildbot/steps/source.py for
-# CVS, SVN, and others.
-
-cvsroot = ":pserver:anonymous@cvs.sourceforge.net:/cvsroot/buildbot"
-cvsmodule = "buildbot"
+from buildbot.process.factory import BuildFactory
+from buildbot.steps.source import Git
+from buildbot.steps.shell import ShellCommand
-from buildbot.process import factory
-from buildbot.steps.source import CVS
-from buildbot.steps.shell import Compile
-from buildbot.steps.python_twisted import Trial
-f1 = factory.BuildFactory()
-f1.addStep(CVS(cvsroot=cvsroot, cvsmodule=cvsmodule, login="", mode="copy"))
-f1.addStep(Compile(command=["python", "./setup.py", "build"]))
-f1.addStep(Trial(testChanges=True, testpath="."))
+factory = BuildFactory()
+# check out the source
+factory.addStep(Git(repourl='git://github.com/buildbot/pyflakes.git', mode='copy'))
+# run the tests (note that this will require that 'trial' is installed)
+factory.addStep(ShellCommand(command=["trial", "pyflakes"]))
from buildbot.config import BuilderConfig
-b1 = BuilderConfig(name="buildbot-full",
- slavename="bot1name",
- builddir="full",
- factory=f1)
-c['builders'] = [b1]
+c['builders'] = []
+c['builders'].append(
+ BuilderConfig(name="runtests",
+ slavenames=["example-slave"],
+ factory=factory))
####### STATUS TARGETS
# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
c['status'] = []
from buildbot.status import html
from buildbot.status.web import auth, authz
authz_cfg=authz.Authz(
# change any of these to True to enable; see the manual for more
# options
gracefulShutdown = False,
- forceBuild = False,
+ forceBuild = True, # use this to test your slave once it is set up
forceAllBuilds = False,
pingBuilder = False,
stopBuild = False,
stopAllBuilds = False,
cancelPendingBuild = False,
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
-# from buildbot.status import mail
-# c['status'].append(mail.MailNotifier(fromaddr="buildbot@localhost",
-# extraRecipients=["builds@example.com"],
-# sendToInterestedUsers=False))
-#
-# from buildbot.status import words
-# c['status'].append(words.IRC(host="irc.example.com", nick="bb",
-# channels=["#example"]))
-# c['status'].append(words.IRC(host="irc.example.com", nick="bb",
-# channels=["#example"], useSSL=True))
-#
-# from buildbot.status import client
-# c['status'].append(client.PBListener(9988))
-
-
-####### DEBUGGING OPTIONS
-
-# if you set 'debugPassword', then you can connect to the buildmaster with
-# the diagnostic tool in contrib/debugclient.py . From this tool, you can
-# manually force builds and inject changes, which may be useful for testing
-# your buildmaster without actually committing changes to your repository (or
-# before you have a functioning 'sources' set up). The debug tool uses the
-# same port number as the slaves do: 'slavePortnum'.
-
-#c['debugPassword'] = "debugpassword"
-
-# if you set 'manhole', you can ssh into the buildmaster and get an
-# interactive python shell, which may be useful for debugging buildbot
-# internals. It is probably only useful for buildbot developers. You can also
-# use an authorized_keys file, or plain telnet.
-#from buildbot import manhole
-#c['manhole'] = manhole.PasswordManhole("tcp:9999:interface=127.0.0.1",
-# "admin", "password")
-
-
####### PROJECT IDENTITY
# the 'projectName' string will be used to describe the project that this
# buildbot is working on. For example, it is used as the title of the
# waterfall HTML page. The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.
-c['projectName'] = "Buildbot"
-c['projectURL'] = "http://buildbot.net/"
+c['projectName'] = "Pyflakes"
+c['projectURL'] = "http://divmod.org/trac/wiki/DivmodPyflakes"
# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.
c['buildbotURL'] = "http://localhost:8010/"
+
+####### DB URL
+
+# This specifies what database buildbot uses to store change and scheduler
+# state. You can leave this at its default for all but the largest
+# installations.
+c['db_url'] = "sqlite:///state.sqlite"
+
--- a/master/buildbot/scripts/startup.py
+++ b/master/buildbot/scripts/startup.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import os, sys, time
class Follower:
def follow(self):
from twisted.internet import reactor
from buildbot.scripts.logwatcher import LogWatcher
self.rc = 0
--- a/master/buildbot/sourcestamp.py
+++ b/master/buildbot/sourcestamp.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_sourcestamp -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from zope.interface import implements
from twisted.persisted import styles
from buildbot import util, interfaces
class SourceStamp(util.ComparableMixin, styles.Versioned):
"""This is a tuple of (branch, revision, patchspec, changes, project, repository).
@@ -20,16 +34,17 @@ class SourceStamp(util.ComparableMixin,
If REV is None, checkout HEAD and patch it.
- (revision=None, patchspec=None, changes=[CHANGES]): let the Source
step check out the latest revision indicated by the given Changes.
CHANGES is a tuple of L{buildbot.changes.changes.Change} instances,
and all must be on the same branch.
"""
persistenceVersion = 2
+ persistenceForgets = ( 'wasUpgraded', )
# all six of these are publically visible attributes
branch = None
revision = None
patch = None
changes = ()
project = ''
repository = ''
@@ -89,26 +104,25 @@ class SourceStamp(util.ComparableMixin,
# be merged. It might be the case that revision==None, so they're
# both building HEAD.
return True
return False
def mergeWith(self, others):
"""Generate a SourceStamp for the merger of me and all the other
- BuildRequests. This is called by a Build when it starts, to figure
+ SourceStamps. This is called by a Build when it starts, to figure
out what its sourceStamp should be."""
# either we're all building the same thing (changes==None), or we're
# all building changes (which can be merged)
changes = []
changes.extend(self.changes)
- for req in others:
- assert self.canBeMergedWith(req) # should have been checked already
- changes.extend(req.changes)
+ for ss in others:
+ changes.extend(ss.changes)
newsource = SourceStamp(branch=self.branch,
revision=self.revision,
patch=self.patch,
project=self.project,
repository=self.repository,
changes=changes)
return newsource
@@ -148,15 +162,17 @@ class SourceStamp(util.ComparableMixin,
def upgradeToVersion1(self):
# version 0 was untyped; in version 1 and later, types matter.
if self.branch is not None and not isinstance(self.branch, str):
self.branch = str(self.branch)
if self.revision is not None and not isinstance(self.revision, str):
self.revision = str(self.revision)
if self.patch is not None:
self.patch = ( int(self.patch[0]), str(self.patch[1]) )
+ self.wasUpgraded = True
def upgradeToVersion2(self):
# version 1 did not have project or repository; just set them to a default ''
self.project = ''
self.repository = ''
+ self.wasUpgraded = True
# vim: set ts=4 sts=4 sw=4 et:
--- a/master/buildbot/status/base.py
+++ b/master/buildbot/status/base.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from zope.interface import implements
from twisted.application import service
from buildbot.interfaces import IStatusReceiver
from buildbot import util, pbutil
class StatusReceiver:
--- a/master/buildbot/status/builder.py
+++ b/master/buildbot/status/builder.py
@@ -1,14 +1,29 @@
-# -*- test-case-name: buildbot.test.test_status -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from zope.interface import implements
from twisted.python import log, runtime
from twisted.persisted import styles
from twisted.internet import reactor, defer, threads
+import twisted.internet.interfaces
from twisted.protocols import basic
from buildbot.process.properties import Properties
from buildbot.util import collections
from buildbot.util.eventual import eventually
import weakref
import os, shutil, re, urllib, itertools
import gc
@@ -20,17 +35,17 @@ from gzip import GzipFile
# sibling imports
from buildbot import interfaces, util, sourcestamp
SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = range(6)
Results = ["success", "warnings", "failure", "skipped", "exception", "retry"]
def worst_status(a, b):
- # SUCCESS > SKIPPED > WARNINGS > FAILURE > EXCEPTION > RETRY
+ # SUCCESS > WARNINGS > FAILURE > EXCEPTION > RETRY
# Retry needs to be considered the worst so that conusmers don't have to
# worry about other failures undermining the RETRY.
for s in (RETRY, EXCEPTION, FAILURE, WARNINGS, SKIPPED, SUCCESS):
if s in (a, b):
return s
# build processes call the following methods:
#
@@ -50,20 +65,36 @@ def worst_status(a, b):
# startBuild
# finishBuild
STDOUT = interfaces.LOG_CHANNEL_STDOUT
STDERR = interfaces.LOG_CHANNEL_STDERR
HEADER = interfaces.LOG_CHANNEL_HEADER
ChunkTypes = ["stdout", "stderr", "header"]
+class NullAddress(object):
+ "an address for NullTransport"
+ implements(twisted.internet.interfaces.IAddress)
+
+class NullTransport(object):
+ "a do-nothing transport to make NetstringReceiver happy"
+ implements(twisted.internet.interfaces.ITransport)
+ def write(self, data): raise NotImplementedError
+ def writeSequence(self, data): raise NotImplementedError
+ def loseConnection(self): pass
+ def getPeer(self):
+ return NullAddress
+ def getHost(self):
+ return NullAddress
+
class LogFileScanner(basic.NetstringReceiver):
def __init__(self, chunk_cb, channels=[]):
self.chunk_cb = chunk_cb
self.channels = channels
+ self.makeConnection(NullTransport())
def stringReceived(self, line):
channel = int(line[0])
if not self.channels or (channel in self.channels):
self.chunk_cb((channel, line[1:]))
class LogFileProducer:
"""What's the plan?
@@ -807,17 +838,19 @@ class BuildStepStatus(styles.Versioned):
@type logs: dict of string -> L{buildbot.status.builder.LogFile}
@ivar logs: logs of steps
@type statistics: dict
@ivar statistics: results from running this step
"""
# note that these are created when the Build is set up, before each
# corresponding BuildStep has started.
implements(interfaces.IBuildStepStatus, interfaces.IStatusEvent)
+
persistenceVersion = 3
+ persistenceForgets = ( 'wasUpgraded', )
started = None
finished = None
progress = None
text = []
results = (None, [])
text2 = []
watchers = []
@@ -996,21 +1029,17 @@ class BuildStepStatus(styles.Versioned):
return log
def addHTMLLog(self, name, html):
assert self.started # addLog before stepStarted won't notify watchers
logfilename = self.build.generateLogfileName(self.name, name)
log = HTMLLogFile(self, name, logfilename, html)
self.logs.append(log)
for w in self.watchers:
- receiver = w.logStarted(self.build, self, log)
- # TODO: think about this: there isn't much point in letting
- # them subscribe
- #if receiver:
- # log.subscribe(receiver, True)
+ w.logStarted(self.build, self, log)
w.logFinished(self.build, self, log)
def logFinished(self, log):
for w in self.watchers:
w.logFinished(self.build, self, log)
def addURL(self, name, url):
self.urls[name] = url
@@ -1085,28 +1114,34 @@ class BuildStepStatus(styles.Versioned):
def __setstate__(self, d):
styles.Versioned.__setstate__(self, d)
# self.build must be filled in by our parent
# point the logs to this object
for loog in self.logs:
loog.step = self
+ self.watchers = []
+ self.finishedWatchers = []
+ self.updates = {}
def upgradeToVersion1(self):
if not hasattr(self, "urls"):
self.urls = {}
+ self.wasUpgraded = True
def upgradeToVersion2(self):
if not hasattr(self, "statistics"):
self.statistics = {}
+ self.wasUpgraded = True
def upgradeToVersion3(self):
if not hasattr(self, "step_number"):
self.step_number = 0
+ self.wasUpgraded = True
def asDict(self):
result = {}
# Constant
result['name'] = self.getName()
# Transient
result['text'] = self.getText()
@@ -1122,17 +1157,19 @@ class BuildStepStatus(styles.Versioned):
# TODO(maruel): Move that to a sub-url or just publish the log_url
# instead.
#result['logs'] = self.getLogs()
return result
class BuildStatus(styles.Versioned):
implements(interfaces.IBuildStatus, interfaces.IStatusEvent)
+
persistenceVersion = 3
+ persistenceForgets = ( 'wasUpgraded', )
source = None
reason = None
changes = []
blamelist = []
progress = None
started = None
finished = None
@@ -1473,25 +1510,28 @@ class BuildStatus(styles.Versioned):
changes = getattr(self, 'changes', [])
source = sourcestamp.SourceStamp(branch=None,
revision=None,
patch=patch,
changes=changes)
self.source = source
self.changes = source.changes
del self.sourceStamp
+ self.wasUpgraded = True
def upgradeToVersion2(self):
self.properties = {}
+ self.wasUpgraded = True
def upgradeToVersion3(self):
# in version 3, self.properties became a Properties object
propdict = self.properties
self.properties = Properties()
self.properties.update(propdict, "Upgrade from previous version")
+ self.wasUpgraded = True
def upgradeLogfiles(self):
# upgrade any LogFiles that need it. This must occur after we've been
# attached to our Builder, and after we know about all LogFiles of
# all Steps (to get the filenames right).
assert self.builder
for s in self.steps:
for l in s.getLogs():
@@ -1574,17 +1614,19 @@ class BuilderStatus(styles.Versioned):
.builder_status attribute.
@type category: string
@ivar category: user-defined category this builder belongs to; can be
used to filter on in status clients
"""
implements(interfaces.IBuilderStatus, interfaces.IEventSource)
+
persistenceVersion = 1
+ persistenceForgets = ( 'wasUpgraded', )
# these limit the amount of memory we consume, as well as the size of the
# main Builder pickle. The Build and LogFile pickles on disk must be
# handled separately.
buildCacheSize = 15
eventHorizon = 50 # forget events beyond this
# these limit on-disk storage
@@ -1648,25 +1690,26 @@ class BuilderStatus(styles.Versioned):
self.watchers = []
self.slavenames = []
# self.basedir must be filled in by our parent
# self.status must be filled in by our parent
def reconfigFromBuildmaster(self, buildmaster):
# Note that we do not hang onto the buildmaster, since this object
# gets pickled and unpickled.
- if buildmaster.buildCacheSize:
+ if buildmaster.buildCacheSize is not None:
self.buildCacheSize = buildmaster.buildCacheSize
def upgradeToVersion1(self):
if hasattr(self, 'slavename'):
self.slavenames = [self.slavename]
del self.slavename
if hasattr(self, 'nextBuildNumber'):
del self.nextBuildNumber # determineNextBuildNumber chooses this
+ self.wasUpgraded = True
def determineNextBuildNumber(self):
"""Scan our directory of saved BuildStatus instances to determine
what our self.nextBuildNumber should be. Set it one larger than the
highest-numbered build we discover. This is called by the top-level
Status object shortly after we are created or loaded from disk.
"""
existing_builds = [int(f)
@@ -1733,18 +1776,29 @@ class BuilderStatus(styles.Versioned):
return self.touchBuildCache(self.buildCache[number])
# then fall back to loading it from disk
filename = self.makeBuildFilename(number)
try:
log.msg("Loading builder %s's build %d from on-disk pickle"
% (self.name, number))
build = load(open(filename, "rb"))
+ build.builder = self
+
+ # (bug #1068) if we need to upgrade, we probably need to rewrite
+ # this pickle, too. We determine this by looking at the list of
+ # Versioned objects that have been unpickled, and (after doUpgrade)
+ # checking to see if any of them set wasUpgraded. The Versioneds'
+ # upgradeToVersionNN methods all set this.
+ versioneds = styles.versionedsToUpgrade
styles.doUpgrade()
- build.builder = self
+ if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]:
+ log.msg("re-writing upgraded build pickle")
+ build.saveYourself()
+
# handle LogFiles from after 0.5.0 and before 0.6.5
build.upgradeLogfiles()
# check that logfiles exist
build.checkLogfiles()
return self.touchBuildCache(build)
except IOError:
raise IndexError("no such build %d" % number)
except EOFError:
@@ -1755,22 +1809,22 @@ class BuilderStatus(styles.Versioned):
self.events = self.events[-self.eventHorizon:]
if events_only:
return
gc.collect()
# get the horizons straight
- if self.buildHorizon:
+ if self.buildHorizon is not None:
earliest_build = self.nextBuildNumber - self.buildHorizon
else:
earliest_build = 0
- if self.logHorizon:
+ if self.logHorizon is not None:
earliest_log = self.nextBuildNumber - self.logHorizon
else:
earliest_log = 0
if earliest_log < earliest_build:
earliest_log = earliest_build
if earliest_build == 0:
@@ -2484,17 +2538,28 @@ class Status:
"""
@rtype: L{BuilderStatus}
"""
filename = os.path.join(self.basedir, basedir, "builder")
log.msg("trying to load status pickle from %s" % filename)
builder_status = None
try:
builder_status = load(open(filename, "rb"))
+
+ # (bug #1068) if we need to upgrade, we probably need to rewrite
+ # this pickle, too. We determine this by looking at the list of
+ # Versioned objects that have been unpickled, and (after doUpgrade)
+ # checking to see if any of them set wasUpgraded. The Versioneds'
+ # upgradeToVersionNN methods all set this.
+ versioneds = styles.versionedsToUpgrade
styles.doUpgrade()
+ if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]:
+ log.msg("re-writing upgraded builder pickle")
+ builder_status.saveYourself()
+
except IOError:
log.msg("no saved status pickle, creating a new one")
except:
log.msg("error while loading status pickle, creating a new one")
log.msg("error follows:")
log.err()
if not builder_status:
builder_status = BuilderStatus(name, category)
--- a/master/buildbot/status/client.py
+++ b/master/buildbot/status/client.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_status -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.spread import pb
from twisted.python import components, log as twlog
from twisted.internet import reactor
from twisted.application import strports
from twisted.cred import portal, checkers
from buildbot import interfaces
--- a/master/buildbot/status/html.py
+++ b/master/buildbot/status/html.py
@@ -1,6 +1,21 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# compatibility wrapper. This is currently the preferred place for master.cfg
# to import from.
from buildbot.status.web.baseweb import WebStatus
_hush_pyflakes = [WebStatus]
--- a/master/buildbot/status/mail.py
+++ b/master/buildbot/status/mail.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_status -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import re
from email.Message import Message
from email.Utils import formatdate
from email.MIMEText import MIMEText
from email.MIMEMultipart import MIMEMultipart
from StringIO import StringIO
@@ -50,16 +64,18 @@ def defaultMessage(mode, name, build, re
result = Results[results]
ss = build.getSourceStamp()
text = ""
if mode == "all":
text += "The Buildbot has finished a build"
elif mode == "failing":
text += "The Buildbot has detected a failed build"
+ elif mode == "warnings":
+ text += "The Buildbot has detected a problem in the build"
elif mode == "passing":
text += "The Buildbot has detected a passing build"
elif mode == "change" and result == 'success':
text += "The Buildbot has detected a restored build"
else:
text += "The Buildbot has detected a new failure"
if ss and ss.project:
project = ss.project
@@ -168,16 +184,17 @@ class MailNotifier(base.StatusReceiverMu
@param subject: a string to be used as the subject line of the message.
%(builder)s will be replaced with the name of the
builder which provoked the message.
@type mode: string (defaults to all)
@param mode: one of:
- 'all': send mail about all builds, passing and failing
- 'failing': only send mail about builds which fail
+ - 'warnings': send mail if builds contain warnings or fail
- 'passing': only send mail about builds which succeed
- 'problem': only send mail about a build which failed
when the previous build passed
- 'change': only send mail about builds who change status
@type builders: list of strings
@param builders: a list of builder names for which mail should be
sent. Defaults to None (send mail for all builds).
@@ -258,17 +275,17 @@ class MailNotifier(base.StatusReceiverMu
assert isinstance(extraRecipients, (list, tuple))
for r in extraRecipients:
assert isinstance(r, str)
# require full email addresses, not User names
assert VALID_EMAIL.search(r), "%s is not a valid email" % r
self.extraRecipients = extraRecipients
self.sendToInterestedUsers = sendToInterestedUsers
self.fromaddr = fromaddr
- assert mode in ('all', 'failing', 'problem', 'change', 'passing')
+ assert mode in ('all', 'failing', 'problem', 'change', 'passing', 'warnings')
self.mode = mode
self.categories = categories
self.builders = builders
self.addLogs = addLogs
self.relayhost = relayhost
self.subject = subject
if lookup is not None:
if type(lookup) is str:
@@ -332,16 +349,18 @@ class MailNotifier(base.StatusReceiverMu
# here is where we actually do something.
builder = build.getBuilder()
if self.builders is not None and name not in self.builders:
return # ignore this build
if self.categories is not None and \
builder.category not in self.categories:
return # ignore this build
+ if self.mode == "warnings" and results == SUCCESS:
+ return
if self.mode == "failing" and results != FAILURE:
return
if self.mode == "passing" and results != SUCCESS:
return
if self.mode == "problem":
if results != FAILURE:
return
prev = build.getPreviousBuild()
@@ -536,17 +555,17 @@ class MailNotifier(base.StatusReceiverMu
client_factory = ssl.ClientContextFactory()
client_factory.method = SSLv3_METHOD
else:
client_factory = None
if self.smtpUser and self.smtpPassword:
useAuth = True
else:
- useAuth = False
+ useAuth = False
sender_factory = ESMTPSenderFactory(
self.smtpUser, self.smtpPassword,
self.fromaddr, recipients, StringIO(s),
result, contextFactory=client_factory,
requireTransportSecurity=self.useTls,
requireAuthentication=useAuth)
--- a/master/buildbot/status/persistent_queue.py
+++ b/master/buildbot/status/persistent_queue.py
@@ -1,13 +1,28 @@
-# -*- test-case-name: buildbot.test.test_persistent_queue -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
try:
# Python 2.4+
from collections import deque
+ assert deque
except ImportError:
deque = None
import os
import pickle
from zope.interface import implements, Interface
--- a/master/buildbot/status/progress.py
+++ b/master/buildbot/status/progress.py
@@ -1,9 +1,23 @@
-# -*- test-case-name: buildbot.test.test_status -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.internet import reactor
from twisted.spread import pb
from twisted.python import log
from buildbot import util
class StepProgress:
"""I keep track of how much progress a single BuildStep has made.
new file mode 100644
--- /dev/null
+++ b/master/buildbot/status/status_gerrit.py
@@ -0,0 +1,135 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+
+"""Push events to gerrit
+
+."""
+
+from buildbot.status.base import StatusReceiverMultiService
+from buildbot.status.builder import Results
+from twisted.internet import reactor
+from twisted.internet.protocol import ProcessProtocol
+
+def defaultReviewCB(builderName, build, result, arg):
+ message = "Buildbot finished compiling your patchset\n"
+ message += "on configuration: %s\n" % builderName
+ message += "The result is: %s\n" % Results[result].upper()
+
+ # message, verified, reviewed
+ return message, (result == 0 or -1), 0
+
+class GerritStatusPush(StatusReceiverMultiService):
+ """Event streamer to a gerrit ssh server."""
+
+ def __init__(self, server, username, reviewCB=defaultReviewCB, port=29418, reviewArg=None,
+ **kwargs):
+ """
+ @param server: Gerrit SSH server's address to use for push event notifications.
+ @param username: Gerrit SSH server's username.
+ @param reviewCB: Callback that is called each time a build is finished, and that is used
+ to define the message and review approvals depending on the build result.
+ @param port: Gerrit SSH server's port.
+ @param reviewArg: Optional argument that is passed to the callback.
+ """
+ StatusReceiverMultiService.__init__(self)
+ # Parameters.
+ self.gerrit_server = server
+ self.gerrit_username = username
+ self.gerrit_port = port
+ self.reviewCB = reviewCB
+ self.reviewArg = reviewArg
+
+ class LocalPP(ProcessProtocol):
+ def __init__(self, status):
+ self.status = status
+
+ def outReceived(self, data):
+ print "gerritout:", data
+
+ def errReceived(self, data):
+ print "gerriterr:", data
+
+ def processEnded(self, status_object):
+ if status_object.value.exitCode:
+ print "gerrit status: ERROR:", status_object
+ else:
+ print "gerrit status: OK"
+
+ def setServiceParent(self, parent):
+ print """Starting up."""
+ StatusReceiverMultiService.setServiceParent(self, parent)
+ self.status = self.parent.getStatus()
+ self.status.subscribe(self)
+
+ def builderAdded(self, name, builder):
+ return self # subscribe to this builder
+
+ def buildFinished(self, builderName, build, result):
+ """Do the SSH gerrit verify command to the server."""
+ repo, git = False, False
+
+ # Gerrit + Repo
+ try:
+ downloads = build.getProperty("repo_downloads")
+ downloaded = build.getProperty("repo_downloaded").split(" ")
+ repo = True
+ except KeyError:
+ pass
+
+ if repo:
+ if downloads and 2 * len(downloads) == len(downloaded):
+ message, verified, reviewed = self.reviewCB(builderName, build, result, self.reviewArg)
+ for i in range(0, len(downloads)):
+ try:
+ project, change1 = downloads[i].split(" ")
+ except ValueError:
+ return # something is wrong, abort
+ change2 = downloaded[2 * i]
+ revision = downloaded[2 * i + 1]
+ if change1 == change2:
+ self.sendCodeReview(project, revision, message, verified, reviewed)
+ else:
+ return # something is wrong, abort
+ return
+
+ # Gerrit + Git
+ try:
+ build.getProperty("gerrit_branch") # used only to verify Gerrit source
+ project = build.getProperty("project")
+ revision = build.getProperty("got_revision")
+ git = True
+ except KeyError:
+ pass
+
+ if git:
+ message, verified, reviewed = self.reviewCB(builderName, build, result, self.reviewArg)
+ self.sendCodeReview(project, revision, message, verified, reviewed)
+ return
+
+ def sendCodeReview(self, project, revision, message=None, verified=0, reviewed=0):
+ command = ["ssh", self.gerrit_username + "@" + self.gerrit_server, "-p %d" % self.gerrit_port,
+ "gerrit", "review", "--project %s" % str(project)]
+ if message:
+ command.append("--message '%s'" % message)
+ if verified:
+ command.extend(["--verified %d" % int(verified)])
+ if reviewed:
+ command.extend(["--code-review %d" % int(reviewed)])
+ command.append(str(revision))
+ print command
+ reactor.spawnProcess(self.LocalPP(self), "ssh", command)
+
+# vim: set ts=4 sts=4 sw=4 et:
--- a/master/buildbot/status/status_push.py
+++ b/master/buildbot/status/status_push.py
@@ -1,22 +1,37 @@
-# -*- test-case-name: buildbot.broken_test.runs.test_status_push -*-
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
"""Push events to an abstract receiver.
Implements the HTTP receiver."""
import datetime
import logging
import os
import urllib
import urlparse
try:
import simplejson as json
+ assert json
except ImportError:
import json
from buildbot.status.base import StatusReceiverMultiService
from buildbot.status.persistent_queue import DiskQueue, IndexedQueue, \
MemoryQueue, PersistentQueue
from buildbot.status.web.status_json import FilterOut
from twisted.internet import defer, reactor
--- a/master/buildbot/status/tinderbox.py
+++ b/master/buildbot/status/tinderbox.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from email.Message import Message
from email.Utils import formatdate
from zope.interface import implements
from twisted.internet import defer
from buildbot import interfaces
--- a/master/buildbot/status/web/about.py
+++ b/master/buildbot/status/web/about.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.status.web.base import HtmlResource
import buildbot
import twisted
import sys
import jinja2
class AboutBuildbot(HtmlResource):
--- a/master/buildbot/status/web/auth.py
+++ b/master/buildbot/status/web/auth.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import os
from zope.interface import Interface, implements
from buildbot.status.web.base import HtmlResource
class IAuth(Interface):
"""Represent an authentication method."""
--- a/master/buildbot/status/web/authz.py
+++ b/master/buildbot/status/web/authz.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.status.web.auth import IAuth
class Authz(object):
"""Decide who can do what."""
knownActions = [
# If you add a new action here, be sure to also update the documentation
# at docs/cfg-statustargets.texinfo
--- a/master/buildbot/status/web/base.py
+++ b/master/buildbot/status/web/base.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import urlparse, urllib, time, re
import os, cgi, sys, locale
import jinja2
from zope.interface import Interface
from twisted.web import resource, static
from twisted.python import log
from buildbot.status import builder
@@ -50,17 +65,17 @@ Fetch custom build properties from the H
Check the names for valid strings, and return None if a problem is found.
Return a new Properties object containing each property found in req.
"""
properties = Properties()
i = 1
while True:
pname = req.args.get("property%dname" % i, [""])[0]
pvalue = req.args.get("property%dvalue" % i, [""])[0]
- if not pname or not pvalue:
+ if not pname:
break
if not re.match(r'^[\w\.\-\/\~:]*$', pname) \
or not re.match(r'^[\w\.\-\/\~:]*$', pvalue):
log.msg("bad property name='%s', value='%s'" % (pname, pvalue))
return None
properties.setProperty(pname, pvalue, "Force Build Form")
i = i + 1
@@ -197,16 +212,34 @@ class HtmlResource(resource.Resource, Co
title = "Buildbot"
addSlash = False # adapted from Nevow
def getChild(self, path, request):
if self.addSlash and path == "" and len(request.postpath) == 0:
return self
return resource.Resource.getChild(self, path, request)
+
+ def content(self, req, context):
+ """
+ Generate content using the standard layout and the result of the C{body}
+ method.
+
+ This is suitable for the case where a resource just wants to generate
+ the body of a page. It depends on another method, C{body}, being
+ defined to accept the request object and return a C{str}. C{render}
+ will call this method and to generate the response body.
+ """
+ body = self.body(req)
+ context['content'] = body
+ template = req.site.buildbot_service.templates.get_template(
+ "empty.html")
+ return template.render(**context)
+
+
def render(self, request):
# tell the WebStatus about the HTTPChannel that got opened, so they
# can close it if we get reconfigured and the WebStatus goes away.
# They keep a weakref to this, since chances are good that it will be
# closed by the browser or by us before we get reconfigured. See
# ticket #102 for details.
if hasattr(request, "channel"):
# web.distrib.Request has no .channel
--- a/master/buildbot/status/web/baseweb.py
+++ b/master/buildbot/status/web/baseweb.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
import os, weakref
from zope.interface import implements
from twisted.python import log
from twisted.application import strports, service
from twisted.web import server, distrib, static
from twisted.spread import pb
--- a/master/buildbot/status/web/build.py
+++ b/master/buildbot/status/web/build.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.web import html
from twisted.web.util import Redirect, DeferredResource
from twisted.internet import defer, reactor
import urllib, time
from twisted.python import log
from buildbot.status.web.base import HtmlResource, \
--- a/master/buildbot/status/web/builder.py
+++ b/master/buildbot/status/web/builder.py
@@ -1,22 +1,36 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.web import html
from twisted.web.util import Redirect
import re, urllib, time
from twisted.python import log
from buildbot import interfaces
from buildbot.status.web.base import HtmlResource, BuildLineMixin, \
path_to_build, path_to_slave, path_to_builder, path_to_change, \
path_to_root, getAndCheckProperties, ICurrentBox, build_get_class, \
map_branches, path_to_authfail
from buildbot.sourcestamp import SourceStamp
-from buildbot.status.builder import BuildRequestStatus
from buildbot.status.web.build import BuildsResource, StatusResourceBuild
from buildbot import util
# /builders/$builder
class StatusResourceBuilder(HtmlResource, BuildLineMixin):
addSlash = True
def __init__(self, builder_status):
@@ -65,28 +79,26 @@ class StatusResourceBuilder(HtmlResource
cxt['pending'] = []
for pb in b.getPendingBuilds():
source = pb.getSourceStamp()
changes = []
if source.changes:
for c in source.changes:
changes.append({ 'url' : path_to_change(req, c),
- 'who' : c.who})
- if source.revision:
- reason = source.revision
- else:
- reason = "no changes specified"
+ 'who' : c.who,
+ 'revision' : c.revision,
+ 'repo' : c.repository })
cxt['pending'].append({
'when': time.strftime("%b %d %H:%M:%S", time.localtime(pb.getSubmitTime())),
'delay': util.formatInterval(util.now() - pb.getSubmitTime()),
- 'reason': reason,
'id': pb.brid,
- 'changes' : changes
+ 'changes' : changes,
+ 'num_changes' : len(changes),
})
numbuilds = int(req.args.get('numbuilds', ['5'])[0])
recent = cxt['recent'] = []
for build in b.generateFinishedBuilds(num_builds=int(numbuilds)):
recent.append(self.get_line_values(req, build, False))
sl = cxt['slaves'] = []
--- a/master/buildbot/status/web/buildstatus.py
+++ b/master/buildbot/status/web/buildstatus.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.status.web.base import HtmlResource, IBox
class BuildStatusStatusResource(HtmlResource):
def __init__(self, categories=None):
HtmlResource.__init__(self)
def content(self, request, ctx):
"""Display a build in the same format as the waterfall page.
--- a/master/buildbot/status/web/change_hook.py
+++ b/master/buildbot/status/web/change_hook.py
@@ -1,23 +1,32 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# code inspired/copied from contrib/github_buildbot
# and inspired from code from the Chromium project
# otherwise, Andrew Melo <andrew.melo@gmail.com> wrote the rest
# but "the rest" is pretty minimal
from twisted.web import resource
-from buildbot.status.builder import FAILURE
import re
-from buildbot import util, interfaces
-import traceback
-import sys
-from buildbot.process.properties import Properties
-from buildbot.changes.changes import Change
from twisted.python.reflect import namedModule
-from twisted.python.log import msg,err
+from twisted.python.log import msg
from buildbot.util import json
class ChangeHookResource(resource.Resource):
# this is a cheap sort of template thingy
contentType = "text/html; charset=utf-8"
children = {}
def __init__(self, dialects={}):
"""
--- a/master/buildbot/status/web/changes.py
+++ b/master/buildbot/status/web/changes.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from zope.interface import implements
from twisted.python import components
from twisted.web.error import NoResource
from buildbot.changes.changes import Change
from buildbot.status.web.base import HtmlResource, IBox, Box
--- a/master/buildbot/status/web/console.py
+++ b/master/buildbot/status/web/console.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from __future__ import generators
import time
import operator
import re
import urllib
from buildbot import util
@@ -344,21 +359,18 @@ class ConsoleStatusResource(HtmlResource
categories = builderList.keys()
categories.sort()
cs = []
for category in categories:
c = {}
- # TODO(nsylvain): Another hack to display the category in a pretty
- # way. If the master owner wants to display the categories in a
- # given order, he/she can prepend a number to it. This number won't
- # be shown.
- c["name"] = category.lstrip('0123456789')
+
+ c["name"] = category
# To be able to align the table correctly, we need to know
# what percentage of space this category will be taking. This is
# (#Builders in Category) / (#Builders Total) * 100.
c["size"] = (len(builderList[category]) * 100) / count
cs.append(c)
return cs
--- a/master/buildbot/status/web/feeds.py
+++ b/master/buildbot/status/web/feeds.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# This module enables ATOM and RSS feeds from webstatus.
#
# It is based on "feeder.py" which was part of the Buildbot
# configuration for the Subversion project. The original file was
# created by Lieven Gobaerts and later adjusted by API
# (apinheiro@igalia.coma) and also here
# http://code.google.com/p/pybots/source/browse/trunk/master/Feeder.py
#
--- a/master/buildbot/status/web/files/default.css
+++ b/master/buildbot/status/web/files/default.css
@@ -353,17 +353,17 @@ div.BuildWaterfall {
}
.warnings {
color: #FFFFFF;
background-color: #ffc343;
border-color: #C29D46;
}
-.exception {
+.exception,.retry {
color: #FFFFFF;
background-color: #f6f;
border-color: #ACA0B3;
}
.start,.running,.waiting,td.building {
color: #666666;
background-color: #ff6;
--- a/master/buildbot/status/web/grid.py
+++ b/master/buildbot/status/web/grid.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from __future__ import generators
from buildbot.status.web.base import HtmlResource
from buildbot.status.web.base import build_get_class, path_to_builder, path_to_build
from buildbot.sourcestamp import SourceStamp
class ANYBRANCH: pass # a flag value, used below
--- a/master/buildbot/status/web/hooks/base.py
+++ b/master/buildbot/status/web/hooks/base.py
@@ -1,25 +1,30 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
# code inspired/copied from contrib/github_buildbot
# and inspired from code from the Chromium project
# otherwise, Andrew Melo <andrew.melo@gmail.com> wrote the rest
# but "the rest" is pretty minimal
-from twisted.web import resource
-from buildbot.status.builder import FAILURE
-import re
-from buildbot import util, interfaces
-import logging
-import traceback
-import sys
-from buildbot.process.properties import Properties
from buildbot.changes.changes import Change
-from twisted.python.reflect import namedModule
from buildbot.util import json
-from twisted.python.log import msg,err
def getChanges(request, options=None):
"""
Consumes a naive build notification (the default for now)
basically, set POST variables to match commit object parameters:
revision, revlink, comments, branch, who, files, links
files, links and properties will be de-json'd, the rest are interpreted as strings
--- a/master/buildbot/status/web/hooks/github.py
+++ b/master/buildbot/status/web/hooks/github.py
@@ -1,41 +1,46 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
#!/usr/bin/env python
"""
github_buildbot.py is based on git_buildbot.py
github_buildbot.py will determine the repository information from the JSON
HTTP POST it receives from github.com and build the appropriate repository.
If your github repository is private, you must add a ssh key to the github
repository for the user who initiated the build on the buildslave.
"""
-import tempfile
import logging
import re
import sys
import traceback
-from twisted.web import server, resource
-from twisted.internet import reactor
-from twisted.spread import pb
-from twisted.cred import credentials
-from optparse import OptionParser
from buildbot.changes.changes import Change
import datetime
-import time
from twisted.python import log
import calendar
-import time
-import calendar
-import datetime
-import re
try:
import json
+ assert json
except ImportError:
import simplejson as json
# python is silly about how it handles timezones
class fixedOffset(datetime.tzinfo):
"""
fixed offset timezone
"""
@@ -81,17 +86,18 @@ def getChanges(request, options = None):
request
the http request object
"""
try:
payload = json.loads(request.args['payload'][0])
user = payload['repository']['owner']['name']
repo = payload['repository']['name']
repo_url = payload['repository']['url']
- private = payload['repository']['private']
+ # This field is unused:
+ #private = payload['repository']['private']
changes = process_change(payload, user, repo, repo_url)
log.msg("Received %s changes from github" % len(changes))
return changes
except Exception:
logging.error("Encountered an exception:")
for msg in traceback.format_exception(*sys.exc_info()):
logging.error(msg.strip())
--- a/master/buildbot/status/web/logs.py
+++ b/master/buildbot/status/web/logs.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from zope.interface import implements
from twisted.python import components
from twisted.spread import pb
from twisted.web import server
from twisted.web.resource import Resource
from twisted.web.error import NoResource
--- a/master/buildbot/status/web/olpb.py
+++ b/master/buildbot/status/web/olpb.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from buildbot.status.web.base import HtmlResource, BuildLineMixin, map_branches
# /one_line_per_build
# accepts builder=, branch=, numbuilds=, reload=
class OneLinePerBuild(HtmlResource, BuildLineMixin):
"""This shows one line per build, combining all builders together. Useful
query arguments:
--- a/master/buildbot/status/web/root.py
+++ b/master/buildbot/status/web/root.py
@@ -1,13 +1,26 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
from twisted.web.util import redirectTo
-from twisted.python import log
-from twisted.internet import reactor
-from buildbot.status.web.base import HtmlResource, path_to_root, path_to_authfail
+from buildbot.status.web.base import HtmlResource, path_to_authfail
from buildbot.util.eventual import eventually
class RootPage(HtmlResource):
title = "Buildbot"
def content(self, request, cxt):
status = self.getStatus(request)
--- a/master/buildbot/status/web/slaves.py
+++ b/master/buildbot/status/web/slaves.py
@@ -1,8 +1,23 @@
+# This file is part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
