warningstep.py
author Benjamin Smedberg <benjamin@smedbergs.us>
Fri, 25 Sep 2009 12:23:53 -0400
branchnewdbschema
changeset 20 5c4f22089101fd8f02dfcf0b2c96544a48f56c61
parent 14 93ae8501dda92da951b959e2f743641aac3f5a2c
child 21 b792fdef1b26fb42fdf3ba7127ed62cb517c97fd
permissions -rw-r--r--
The new DB schema, but slow. Checkpoint.

from buildbot.process.buildstep import SUCCESS, FAILURE
from buildbot.steps.shell import ShellCommand
import MySQLdb, os.path, sys

class Location(object):
    def __init__(self, type, file, lineno, rev, who):
        self.type = type
        self.file = file
        self.lineno = lineno
        self.rev = rev
        self.who = who

    def getLocationID(self, db, cur):
        cur.execute('''INSERT INTO locations (ltype, lfile, lline, lrev, lwho)
                       VALUES (%s, %s, %s, %s, %s)
                       ON DUPLICATE KEY UPDATE locationid = LAST_INSERT_ID(locationid)''',
                    (self.type, self.file, self.lineno, self.rev, self.who))
        print "locationid: %i" % (db.insert_id(),)
        return db.insert_id()

class WarningLine(object):
    def __init__(self, data):
        file, lineno, msg, blametype, blamewho, blamefile, blamerev, blameline = data

        if blametype is None:
            blametype = 'system'
        if lineno is None:
            lineno = -1
        if blamewho is None:
            blamewho = ''
        if blamefile is None:
            blamefile = file.rsplit('/', 1)[-1]
        if blameline is None:
            blameline = lineno
        if blamerev is None:
            blamerev = ''

        self.blame = Location(blametype, blamefile, blameline, blamerev, blamewho)
        self.loc = Location('build', file, lineno, '', '')
        self.msg = msg

    def getMessageID(self, db, cur):
        cur.execute('''INSERT INTO messages (msg) VALUES (%s)
                       ON DUPLICATE KEY UPDATE msgid = LAST_INSERT_ID(msgid)''',
                    (self.msg,))
        print "msgid: %i" % (db.insert_id(),)
        return db.insert_id()

class Warning(object):
    def __init__(self):
        self.lines = []

    def append(self, wl):
        self.lines.append(wl)

    def findWarningID(self, db, cur):
        cols = 'linecount, primaryline, ' + ','.join(['l%i, msg%i' % (i, i) for i in range(0, len(self.lines))])
        values = '%s, 0, ' + ','.join(['%s, %s' for i in range(0, len(self.lines))])
        p = [len(self.lines)]
        for i in range(0, len(self.lines)):
            p += [self.lines[i].blame.getLocationID(db, cur), self.lines[i].getMessageID(db, cur)]

        q = '''INSERT INTO warnings (%s) VALUES (%s)
               ON DUPLICATE KEY UPDATE warningid = LAST_INSERT_ID(warningid)''' % (cols, values)
        cur.execute(q, p)
        print "warningid: %i" % (db.insert_id(),)
        return db.insert_id()

    def insertIntoBuild(self, buildnumber, warningid, db, cur):
        cur.execute('''INSERT INTO buildwarnings (buildnumber, warningid) VALUES (%s, %s)
                       ON DUPLICATE KEY UPDATE wcount = wcount + 1''', (buildnumber, warningid))
        if cur.rowcount == 1: # New row, not DUPLICATE KEY
            p = [[buildnumber, warningid, i, self.lines[i].loc.getLocationID(db, cur)] for i in range(0, len(self.lines))]
            cur.executemany('''INSERT INTO buildwarninglines (buildnumber, warningid, wline, locationid)
                               VALUES (%s, %s, %s, %s)''', p)

class WarningParserStep(ShellCommand):
    """
    A subclass of shellcommand used for running warning-parser.py.

    This parses the output and saves interesting information to the
    master-side sqlite database. It produces output about "new" warnings, etc.

    If the dbfile does not exist, it will be created the first time it is needed.
    """

    def __init__(self, connectargs, **kwargs):
        ShellCommand.__init__(self, **kwargs)

        self.connectargs = connectargs
        self.addFactoryArguments(connectargs=connectargs)

    def createSummary(self, log):
        stepresults = self.step_status.getBuild().getSteps()
        for sr in stepresults:
            result, str = sr.getResults()
            if result == FAILURE:
                self.addCompleteLog('skipped', 'Skipped analyzing warnings because a prior step failed.')
                return

        comparison = self.parseLog(log.getText(), self.getProperty('buildnumber'), self.getProperty('got_revision'))
        self.addCompleteLog('comparison', '\n'.join(comparison))

    def parseLog(self, logdata, buildnumber, revision):
        # Use different connections for the warningID-finder (which does table locking and can't have a long-lived transaction),
        # and the buildwarnings inserter, which should have a long-lived transaction
        warndb = MySQLdb.connect(**self.connectargs)
        warndb.autocommit(True)
        warncur = warndb.cursor()

        db = MySQLdb.connect(**self.connectargs)
        dbcur = db.cursor()

        dbcur.execute('INSERT INTO builds (buildnumber, rev) VALUES ( %s, %s )',
                      (buildnumber, revision))

        def processWarning(w):
            warningid = w.findWarningID(warndb, warncur)
            w.insertIntoBuild(buildnumber, warningid, db, dbcur)

        curwarning = None
        for line in logdata.split("\n"):
            if line.startswith('WARN-DB: '):
                data = eval(line[9:].strip())
                id, ord = data[0:2]
                wl = WarningLine(data[2:])

                if ord == 0:
                    if curwarning is not None:
                        print "processing #%s" % id
                        processWarning(curwarning)
                    curwarning = Warning()

                curwarning.append(wl)

        if curwarning is not None:
            processWarning(curwarning)

        db.commit()

        # A list of line to stick in the comparison log
        comparison = []

        dbcur.execute('''SELECT COUNT(*) as unqiue, SUM(wcount) AS total
                       FROM buildwarnings
                       WHERE buildnumber = %s''', (buildnumber,))
        unique, total = cur.fetchone()
        comparison.append('TinderboxPrint:<a href="http://office.smedbergs.us:8080/build?id=%i">warn:%i</a>' % (buildnumber, unique))
        comparison.append('TinderboxPrint:(%s total)' % (total))

        dbcur.execute('SELECT MAX(buildnumber) FROM builds WHERE buildnumber < %s', (buildnumber,))
        r = dbcur.fetchone()
        if r is None:
            comparison.append('no prior build found: skipping comparison')
        else:
            prevbuildnumber, = r

            cbcur.execute('''SELECT COUNT(*)
                             FROM buildwarnings
                             WHERE buildnumber = %s AND
                               NOT EXISTS (SELECT *
                                           FROM buildwarnings AS oldwarnings
                                           WHERE oldwarnings.warningid = warnings.warningid AND
                                             oldwarnings.buildnumber = %s)''', (buildnumber, prevbuildnumber))
            wnewcount, = cur.fetchone()
            if wnewcount > 0:
                comparison.append('TinderboxPrint:<a href="http://office.smedbergs.us:8080/build?id=%i">warn-new:%i</a>' % (buildnumber, wnewcount))

        dbcur.close()
        db.close()

        return comparison

    def evaluateCommand(self, cmd):
        if cmd.rc != 0:
            return FAILURE
        return SUCCESS