Adding mozilla-specific changes
authorChris AtLee <catlee@mozilla.com>
Tue, 17 Aug 2010 18:40:30 -0400
changeset 81 ebdabf279a4c23e81ff1be2753ee67a23dbcafb5
parent 80 8daaf04dc6a96c009625ab39e1a2da677c357577
child 82 298a2f5cfb3e7b1fea4356fa63ade653eabc331a
push id31
push usercatlee@mozilla.com
push dateThu, 26 Aug 2010 12:50:22 +0000
Adding mozilla-specific changes
buildbot/slave/commands/mozilla.py
master/buildbot/process/builder.py
master/buildbot/status/builder.py
master/buildbot/status/tinderbox.py
slave/buildslave/bot.py
slave/buildslave/commands/mozilla.py
--- a/master/buildbot/process/builder.py
+++ b/master/buildbot/process/builder.py
@@ -461,17 +461,16 @@ class Builder(pb.Referenceable, service.
         BuildRequest that I can handle, I will claim the BuildRequest and
         start the build. When the build finishes, I will retire the
         BuildRequest.
         """
         # overall plan:
         #  move .expectations to DB
 
         assert self.running
-        log.msg("Builder.run %s: %s" % (self, self.slaves))
         self.run_count += 1
 
         available_slaves = [sb for sb in self.slaves if sb.isAvailable()]
         if not available_slaves:
             self.updateBigStatus()
             return
         d = self.db.runInteraction(self._claim_buildreqs, available_slaves)
         d.addCallback(self._start_builds)
--- a/master/buildbot/status/builder.py
+++ b/master/buildbot/status/builder.py
@@ -1709,22 +1709,25 @@ class BuilderStatus(styles.Versioned):
             # check that logfiles exist
             build.checkLogfiles()
             return self.touchBuildCache(build)
         except IOError:
             raise IndexError("no such build %d" % number)
         except EOFError:
             raise IndexError("corrupted build pickle %d" % number)
 
-    def prune(self):
-        gc.collect()
-
+    def prune(self, events_only=False):
         # begin by pruning our own events
         self.events = self.events[-self.eventHorizon:]
 
+        if events_only:
+            return
+
+        gc.collect()
+
         # get the horizons straight
         if self.buildHorizon:
             earliest_build = self.nextBuildNumber - self.buildHorizon
         else:
             earliest_build = 0
 
         if self.logHorizon:
             earliest_log = self.nextBuildNumber - self.logHorizon
@@ -1912,26 +1915,28 @@ class BuilderStatus(styles.Versioned):
 
     def addEvent(self, text=[]):
         # this adds a duration event. When it is done, the user should call
         # e.finish(). They can also mangle it by modifying .text
         e = Event()
         e.started = util.now()
         e.text = text
         self.events.append(e)
+        self.prune(events_only=True)
         return e # they are free to mangle it further
 
     def addPointEvent(self, text=[]):
         # this adds a point event, one which occurs as a single atomic
         # instant of time.
         e = Event()
         e.started = util.now()
         e.finished = 0
         e.text = text
         self.events.append(e)
+        self.prune(events_only=True)
         return e # for consistency, but they really shouldn't touch it
 
     def setBigState(self, state):
         needToUpdate = state != self.currentBigState
         self.currentBigState = state
         if needToUpdate:
             self.publishState()
 
--- a/master/buildbot/status/tinderbox.py
+++ b/master/buildbot/status/tinderbox.py
@@ -5,17 +5,17 @@ from email.Utils import formatdate
 from zope.interface import implements
 from twisted.internet import defer
 
 from buildbot import interfaces
 from buildbot.status import mail
 from buildbot.status.builder import SUCCESS, WARNINGS
 from buildbot.steps.shell import WithProperties
 
-import zlib, bz2, base64
+import gzip, bz2, base64, re, cStringIO
 
 # TODO: docs, maybe a test of some sort just to make sure it actually imports
 # and can format email without raising an exception.
 
 class TinderboxMailNotifier(mail.MailNotifier):
     """This is a Tinderbox status notifier. It can send e-mail to a number of
     different tinderboxes or people. E-mails are sent at the beginning and
     upon completion of each build. It can be configured to send out e-mails
@@ -183,29 +183,62 @@ class TinderboxMailNotifier(mail.MailNot
         # if the build finished...
         else:
             text += "%s binaryurl: %s\n" % (t, self.binaryURL)
             text += "%s logcompression: %s\n" % (t, self.logCompression)
 
             # logs will always be appended
             logEncoding = ""
             tinderboxLogs = ""
-            for log in build.getLogs():
-                l = ""
-                if self.logCompression == "bzip2":
-                    compressedLog = bz2.compress(log.getText())
-                    l = base64.encodestring(compressedLog)
-                    logEncoding = "base64";
-                elif self.logCompression == "gzip":
-                    compressedLog = zlib.compress(log.getText())
-                    l = base64.encodestring(compressedLog)
-                    logEncoding = "base64";
-                else:
-                    l = log.getText()
-                tinderboxLogs += l
+            for bs in build.getSteps():
+                # Make sure that shortText is a regular string, so that bad
+                # data in the logs don't generate UnicodeDecodeErrors
+                shortText = "%s\n" % ' '.join(bs.getText()).encode('ascii', 'replace')
+
+                # ignore steps that haven't happened
+                if not re.match(".*[^\s].*", shortText):
+                    continue
+                # we ignore TinderboxPrint's here so we can do things like:
+                # ShellCommand(command=['echo', 'TinderboxPrint:', ...])
+                if re.match(".+TinderboxPrint.*", shortText):
+                    shortText = shortText.replace("TinderboxPrint",
+                                                  "Tinderbox Print")
+                logs = bs.getLogs()
+
+                tinderboxLogs += "======== BuildStep started ========\n"
+                tinderboxLogs += shortText
+                tinderboxLogs += "=== Output ===\n"
+                for log in logs:
+                    logText = log.getTextWithHeaders()
+                    # Because we pull in the log headers here we have to ignore
+                    # some of them. Basically, if we're TinderboxPrint'ing in
+                    # a ShellCommand, the only valid one(s) are at the start
+                    # of a line. The others are prendeded by whitespace, quotes,
+                    # or brackets/parentheses
+                    for line in logText.splitlines():
+                        if re.match(".+TinderboxPrint.*", line):
+                            line = line.replace("TinderboxPrint",
+                                                "Tinderbox Print")
+                        tinderboxLogs += line + "\n"
+
+                tinderboxLogs += "=== Output ended ===\n"
+                tinderboxLogs += "======== BuildStep ended ========\n"
+
+            if self.logCompression == "bzip2":
+                cLog = bz2.compress(tinderboxLogs)
+                tinderboxLogs = base64.encodestring(cLog)
+                logEncoding = "base64"
+            elif self.logCompression == "gzip":
+                cLog = cStringIO.StringIO()
+                gz = gzip.GzipFile(mode="w", fileobj=cLog)
+                gz.write(tinderboxLogs)
+                gz.close()
+                cLog = cLog.getvalue()
+                tinderboxLogs = base64.encodestring(cLog)
+                logEncoding = "base64"
 
             text += "%s logencoding: %s\n" % (t, logEncoding)
             text += "%s END\n\n" % t
             text += tinderboxLogs
             text += "\n"
 
         m = Message()
         m.set_payload(text)
--- a/slave/buildslave/bot.py
+++ b/slave/buildslave/bot.py
@@ -11,16 +11,19 @@ import buildslave
 from buildslave.util import now
 from buildslave.pbutil import ReconnectingPBClientFactory
 from buildslave.commands import registry
 
 # make sure the standard commands get registered. This import is performed
 # for its side-effects.
 from buildslave.commands import base, transfer, vcs
 
+# Special mozilla commands
+from buildslave.commands import mozilla
+
 class NoCommandRunning(pb.Error):
     pass
 class WrongCommandRunning(pb.Error):
     pass
 class UnknownCommand(pb.Error):
     pass
 
 class Master:
rename from buildbot/slave/commands/mozilla.py
rename to slave/buildslave/commands/mozilla.py
--- a/buildbot/slave/commands/mozilla.py
+++ b/slave/buildslave/commands/mozilla.py
@@ -1,15 +1,15 @@
 import os, platform, sha, sys
 from ConfigParser import ConfigParser
 from os import listdir, path
 from stat import ST_SIZE
 
-from buildbot.slave.commands.base import Command, command_version
-from buildbot.slave.commands.registry import registerSlaveCommand
+from buildslave.commands.base import Command, command_version
+from buildslave.commands.registry import registerSlaveCommand
 
 #### Mozilla Slave Commands
 class SetMozillaBuildProperties(Command):
     def setup(self, args):
         self.objdir = args['objdir']
 
     def getPlatform(self):
         # TODO: this should really be part of SlaveBuilder - or something