warningstep.py
author Benjamin Smedberg <benjamin@smedbergs.us>
Fri, 19 Dec 2008 17:03:02 -0500
changeset 9 1e34d76ac686b16ccd215816c8c4abaacc037156
parent 6 03664a1472ce9af391d9b7ad4cea17843672be35
child 10 80fc943e21cf1c1d6ea1a788ae284d737c800eed
permissions -rw-r--r--
Link the tinderbox output to the database frontend.

from buildbot.process.buildstep import SUCCESS, FAILURE
from buildbot.steps.shell import ShellCommand
import sqlite3, os.path

class WarningParserStep(ShellCommand):
    """
    A subclass of shellcommand used for running warning-parser.py.

    This parses the output and saves interesting information to the
    master-side sqlite database. It produces output about "new" warnings, etc.

    If the dbfile does not exist, it will be created the first time it is needed.
    """

    _createSchema = """
    CREATE TABLE builds (
      buildnumber INTEGER NOT NULL PRIMARY KEY,
      rev TEXT NOT NULL
    );

    CREATE TABLE warnings (
      buildnumber INTEGER NOT NULL CONSTRAINT warnings_buildnumer_fkey REFERENCES builds (buildnumber),
      signature TEXT NOT NULL,
      count INTEGER NOT NULL,
      CONSTRAINT warnings_pkey PRIMARY KEY (buildnumber, signature)
    );

    CREATE TABLE wlines (
      buildnumber INTEGER NOT NULL,
      signature TEXT NOT NULL,
      wline INTEGER NOT NULL,
      file TEXT NOT NULL,
      lineno INTEGER,
      msg TEXT NOT NULL,
      blametype TEXT,
      blamefile TEXT,
      blamerev TEXT,
      blameline INTEGER,
      blamewho TEXT,
      CONSTRAINT wlines_refwarning_fkey FOREIGN KEY (buildnumber, signature) REFERENCES warnings (buildnumber, signature),
      CONSTRAINT wlines_pkey PRIMARY KEY (buildnumber, signature, wline)
    );
    """

    def __init__(self, dbfile, **kwargs):
        ShellCommand.__init__(self, **kwargs)

        self.dbfile = dbfile
        self.addFactoryArguments(dbfile=dbfile)

    def createSummary(self, log):
        stepresults = self.step_status.getBuild().getSteps()
        for sr in stepresults:
            result, str = sr.getResults()
            if result == FAILURE:
                self.addCompleteLog('skipped', 'Skipped analyzing warnings because a prior step failed.')
                return

        createSchema = not os.path.exists(self.dbfile)

        db = sqlite3.connect(self.dbfile)
        cur = db.cursor()

        if createSchema:
            cur.executescript(self._createSchema)

        buildnumber = self.getProperty('buildnumber')

        cur.execute('INSERT INTO builds (buildnumber, rev) VALUES ( ?, ? )',
                    (buildnumber, self.getProperty('got_revision')))

        def lineSignature(line):
            """
            Given a line tuple, return a string signature for the line. This signature is based
            on the blame location if available, else the reported location.
            """
            file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline = line

            if blametype is None:
                return "%s:%s:%s" % (file, lineno, msg[:30])

            return "%s:%s:%s:%s:%s" % (blametype, blamepath, blamerev[:12], blameline, msg[:30])

        def processWarning(lines):
            """
            process a warning. A warning is a list of lines. Each line is a tuple of
            (file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline)
            """

            signature = "\t".join( (lineSignature(line) for line in lines) )

            cur.execute('''UPDATE warnings
                           SET count = count + 1
                           WHERE buildnumber = ? AND signature = ?''', (buildnumber, signature))
            if cur.rowcount == 0:
                cur.execute('INSERT INTO warnings (buildnumber, signature, count) VALUES (?, ?, ?)', (buildnumber, signature, 1))

                for i in xrange(0, len(lines)):
                    file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline = lines[i]
                    cur.execute('''INSERT INTO wlines (buildnumber, signature, wline, file, lineno, msg, blametype, blamefile, blamerev, blameline, blamewho)
                                   VALUES             (?,           ?,         ?,     ?,    ?,      ?,   ?,         ?,         ?,        ?,         ?)''',
                                (buildnumber, signature, i, file, lineno, msg, blametype, blamepath, blamerev, blameline, blamewho))


        curwarning = None
        for line in log.getText().split("\n"):
            if line.startswith('WARN-DB: '):
                id, ord, file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline = eval(line[9:].strip())

                if ord == 0:
                    if curwarning is not None:
                        processWarning(curwarning)
                    curwarning = []

                curwarning.append( (file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline) )

        if curwarning is not None:
            processWarning(curwarning)

        db.commit()

        # A list of line to stick in the comparison log
        comparison = []

        cur.execute('SELECT sum(count) FROM warnings WHERE buildnumber = ?', (buildnumber,))
        wcount, = cur.fetchone()
        comparison.append('<a href="http://office.smedbergs.us:8080/build?id=%i">TinderboxPrint:warn:%i</a>' % (buildnumber, wcount))

        cur.execute('SELECT max(buildnumber) FROM builds WHERE buildnumber < ?', (buildnumber,))
        r = cur.fetchone()
        if r is None:
            comparison.append('no prior build found: skipping comparison')
        else:
            prevbuildnumber, = r

            cur.execute('''SELECT signature
                           FROM warnings
                           WHERE buildnumber = ? AND
                             NOT EXISTS (SELECT *
                                         FROM warnings AS oldwarnings
                                         WHERE oldwarnings.signature = warnings.signature AND
                                           oldwarnings.buildnumber = ?)''', (buildnumber, prevbuildnumber))
            cur2 = db.cursor()

            wnewcount = 0

            for signature, in cur:
                wnewcount = wnewcount + 1
                cur2.execute('''SELECT file, lineno, msg, blametype, blamefile, blamerev, blameline, blamewho
                                FROM wlines
                                WHERE buildnumber = ? AND signature = ?
                                ORDER BY wline''',
                             (buildnumber, signature))
                comparison.append('NEW WARNING:')

                for file, lineno, msg, blametype, blamefile, blamerev, blameline, blamewho in cur2:
                    out = "  %s" % file
                    if lineno is not None:
                        out += ":%i" % lineno

                    out += ": %s" % msg

                    if blamewho is not None:
                        out += ' - Blamed on %s' % blamewho

                    if blametype == 'cvs':
                        out += ' -  http://bonsai.mozilla.org/cvsblame.cgi?file=mozilla/%s&rev=HG_REPO_INITIAL_IMPORT&mark=%s#%s' % (blamefile, blameline, blameline)
                    elif blametype == 'hg':
                        out += ' - http://hg.mozilla.org/mozilla-central/annotate/%s/%s#l%i' % (blamerev, blamefile, blameline)

                    comparison.append(out)

        if wnewcount > 0:
            comparison.append("TinderboxPrint:warn-new:%i" % wnewcount)

        self.addCompleteLog('comparison', '\n'.join(comparison))

        cur.close()
        db.close()

    def evaluateCommand(self, cmd):
        if cmd.rc != 0:
            return FAILURE
        return SUCCESS