Merge back to default
authorBenjamin Smedberg <benjamin@smedbergs.us>
Tue, 29 Sep 2009 13:08:31 -0400
changeset 23 8894b0ca1189
parent 19 f2f8dbf94a36 (current diff)
parent 22 4c1458dcdab3 (diff)
child 24 19e7a98a8dc4
push id13
push userbsmedberg@mozilla.com
push date2009-09-29 17:12 +0000
Merge back to default
--- a/master.cfg
+++ b/master.cfg
@@ -102,17 +102,17 @@ changesetLink = '<a href=http://hg.mozil
 f1.addStep(ShellCommand(command=['printf', 'TinderboxPrint:%s\n',
                                  WithProperties(changesetLink)]))
 
 f1.addStep(FileDownload(mastersrc="mozconfig", slavedest=".mozconfig"))
 
 f1.addStep(Compile(command='make -f client.mk -w >build.log 2>&1', env={'LANG': 'C'}, logfiles={'build': 'build.log'}, timeout=1800))
 f1.addStep(FileDownload(mastersrc="warning-parser.py", slavedest="warning-parser.py"))
 f1.addStep(FileDownload(mastersrc="cvsblame.db", slavedest="cvsblame.db"))
-f1.addStep(WarningParserStep(command=['python', 'warning-parser.py', '.', 'cvsblame.db', 'build.log'], dbfile='warnings.db'))
+f1.addStep(WarningParserStep(command=['python', 'warning-parser.py', '.', 'cvsblame.db', 'build.log'], connectargs=slaves.dbinfo))
 
 b1 = {'name': 'static-analysis-bsmedberg',
       'slavename': "staticanalysis-slave-linux",
       'builddir': "full",
       'factory': f1,
       }
 
 f2 = factory.BuildFactory()
--- a/warning-parser.py
+++ b/warning-parser.py
@@ -5,26 +5,26 @@ Reads a build log on stdin. Parse warnin
 and store them in a sqlite database for later consumption.
 
 Uses gmake "Entering directory" and "Leaving directory" messages to
 keep track of the working directory. Resolves relative file paths against
 these working directories, and also follows symlinks back into the
 source tree.
 """
 
-import sys, os, sqlite3, re, mercurial.hg, mercurial.ui, mercurial.revlog, mercurial.node
+import sys, os, re, mercurial.hg, mercurial.ui, mercurial.revlog, mercurial.node
 
 (srcdir, blamedb, logfile) = sys.argv[1:]
 
 srcdir = os.path.realpath(srcdir) + '/'
 
 blamedb = sqlite3.connect(blamedb)
 blamecur = blamedb.cursor()
 
-repo = mercurial.hg.repository(mercurial.ui.ui(), srcdir)
+repo = mercurial.hg.repository(mercurial.ui.ui(quiet=True, interactive=False), srcdir)
 ctx = repo.changectx('.')
 
 emptyBlame = (None, None, None, None, None)
 
 def getCVSBlame(file, line):
     """
     Get CVS blame information for a specific line that is blamed on the initial
     import of CVS code to Mercurial.
--- a/warning-ui/build.html
+++ b/warning-ui/build.html
@@ -15,37 +15,37 @@
     </div>
     <div class="h2block" py:if="previd is not None and prevrev != rev">
       <h2>Changes since <a href="/build?id=${previd}">last build</a>:</h2>
       <a href="http://hg.mozilla.org/mozilla-central/pushloghtml?fromchange=${prevrev}&amp;tochange=${rev}">Log</a>
     </div>
     <div class="h2block">
       <h2>Unique warnings:</h2> ${unique}
     </div>
-    <div class="h2block">
+    <div class="hsblock">
       <h2>Total warnings:</h2> ${total}
     </div>
 
     <py:if test="previd is not None">
       <div py:if="len(newwarnings)">
         <h3>New:</h3>
 
         <ul>
-          <li py:for="signature, file, lineno, msg in newwarnings">
-            <a href="${genlink('warning', signature=signature)}">${file}:${lineno} - ${msg}</a>
+          <li py:for="id, file, lineno, msg in newwarnings">
+            <a href="${genlink('warning', id=id)}">${file}:${lineno} - ${msg}</a>
           </li>
         </ul>
       </div>
 
       <div py:if="len(fixedwarnings)">
         <h3>Fixed:</h3>
 
         <ul>
-          <li py:for="signature, file, lineno, msg in fixedwarnings">
-            <a href="${genlink('warning', signature=signature)}">${file}:${lineno} - ${msg}</a>
+          <li py:for="id, file, lineno, msg in fixedwarnings">
+            <a href="${genlink('warning', id=id)}">${file}:${lineno} - ${msg}</a>
           </li>
         </ul>
       </div>
     </py:if>
 
     <h2>Search warnings:</h2>
 
     ${searchform(id, '', '', 'unused variable%')}
new file mode 100644
--- /dev/null
+++ b/warning-ui/dbschema.sql
@@ -0,0 +1,295 @@
+DROP TABLE IF EXISTS buildwarninglines, buildwarnings, builds, warninglines, warnings, locations, messages;
+
+CREATE TABLE locations (
+  locationid INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  ltype VARCHAR(10) NOT NULL, -- 'cvs', 'hg', 'system' (will not have directory) or 'build'
+  lfile VARCHAR(255) NOT NULL,
+  lline INTEGER NOT NULL, -- use -1 for unknown
+  lrev VARCHAR(40) NOT NULL,
+  lwho VARCHAR(400) NOT NULL,
+  CONSTRAINT location_uniq UNIQUE (ltype, lfile, lline, lrev)
+) ENGINE=InnoDB;
+CREATE INDEX idx_locations_lfile
+  ON locations (lfile);
+INSERT INTO locations
+  (locationid, ltype, lfile, lline, lrev, lwho)
+VALUES
+  (-1,     'invalid', '',    -1,    '',   '');
+
+CREATE TABLE messages (
+  msgid INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  msg VARCHAR(255) NOT NULL UNIQUE
+) ENGINE=InnoDB;
+INSERT INTO messages
+  (msgid, msg)
+VALUES
+  (-1, 'warningdb sentinel message');
+
+CREATE TABLE warnings (
+  warningid INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  linecount TINYINT NOT NULL,
+  primaryline TINYINT NOT NULL
+) ENGINE=InnoDB;
+
+CREATE TABLE warninglines (
+  warningid INTEGER NOT NULL,
+  wline TINYINT NOT NULL,
+  CONSTRAINT warninglines_pkey PRIMARY KEY (warningid, wline),
+  CONSTRAINT warninglines_warningid_fkey
+    FOREIGN KEY (warningid) REFERENCES warnings (warningid),
+  locationid INTEGER NOT NULL REFERENCES locations (locationid),
+  msgid INTEGER NOT NULL REFERENCES messages (msgid)
+) ENGINE=InnoDB;
+CREATE INDEX idx_warninglines_wline ON warninglines (wline, locationid, msgid);
+
+CREATE OR REPLACE VIEW v_warninglines
+AS
+  SELECT warningid, wline, ltype, lfile, lline, lrev, lwho, msg
+  FROM warninglines, locations, messages
+  WHERE locations.locationid = warninglines.locationid
+    AND messages.msgid = warninglines.msgid;
+
+CREATE OR REPLACE VIEW v_warnings
+AS
+  SELECT warnings.warningid, linecount, primaryline, ltype, lfile, lline, lrev, lwho, msg
+  FROM warnings, warninglines, locations, messages
+  WHERE warninglines.warningid = warnings.warningid
+    AND warninglines.wline = warnings.primaryline
+    AND locations.locationid = warninglines.locationid
+    AND messages.msgid = warninglines.msgid;
+
+CREATE TABLE builds (
+  buildnumber INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  rev VARCHAR(40) NOT NULL
+) ENGINE=InnoDB;
+
+CREATE TABLE buildwarnings (
+  buildnumber INTEGER NOT NULL,
+  CONSTRAINT buildwarnings_buildnumber_fkey FOREIGN KEY (buildnumber)
+    REFERENCES builds (buildnumber),
+  warningid INTEGER NOT NULL,
+  CONSTRAINT buildwarnings_pkey PRIMARY KEY (buildnumber, warningid),
+  wcount INTEGER NOT NULL DEFAULT 1
+)  ENGINE=InnoDB ;
+CREATE INDEX idx_buildwarnings_warningid
+  ON buildwarnings (warningid);
+
+CREATE TABLE buildwarninglines (
+  buildnumber INTEGER NOT NULL,
+  warningid INTEGER NOT NULL,
+  wline TINYINT NOT NULL,
+  CONSTRAINT buildwarninglines_warningid_fkey FOREIGN KEY (buildnumber, warningid)
+    REFERENCES buildwarnings (buildnumber, warningid),
+  CONSTRAINT buildwarninglines_pkey PRIMARY KEY (buildnumber, warningid, wline),
+  CONSTRAINT buildwarninglines_wline_fkey FOREIGN KEY (warningid, wline)
+    REFERENCES warninglines (warningid, wline),
+  locationid INTEGER NOT NULL REFERENCES locations (locationid)
+)  ENGINE=InnoDB ;
+
+CREATE OR REPLACE VIEW v_buildwarninglines
+AS
+  SELECT buildnumber, warningid, wline, ltype, lfile, lline, lrev, lwho
+  FROM buildwarninglines, locations
+  WHERE locations.locationid = buildwarninglines.locationid;
+
+CREATE OR REPLACE VIEW v_buildwarnings
+AS
+  SELECT v_buildwarninglines.buildnumber AS buildnumber,
+         v_buildwarninglines.warningid AS warningid,
+         v_buildwarninglines.lfile AS lfile,
+         v_buildwarninglines.lline AS lline,
+         msg, primaryline
+  FROM v_buildwarninglines, buildwarnings, v_warnings
+  WHERE buildwarnings.warningid = v_buildwarninglines.warningid
+    AND v_warnings.warningid = v_buildwarninglines.warningid
+    AND v_buildwarninglines.buildnumber = buildwarnings.buildnumber
+    AND v_buildwarninglines.wline = v_warnings.primaryline;
+
+delimiter /;
+
+DROP PROCEDURE IF EXISTS setuptables;
+CREATE PROCEDURE setuptables()
+MODIFIES SQL DATA
+BEGIN
+  CREATE TEMPORARY TABLE _BUILDDATA (
+    id INTEGER NOT NULL,
+    ord INTEGER NOT NULL,
+    CONSTRAINT PRIMARY KEY (id, ord),
+    wcount INTEGER NOT NULL DEFAULT 1,
+    file VARCHAR(255) NOT NULL,
+    line INTEGER,
+    msg VARCHAR(255) NOT NULL,
+    blametype VARCHAR(10),
+    blamewho VARCHAR(400),
+    blamefile VARCHAR(255),
+    blamerev VARCHAR(40),
+    blameline INTEGER);
+  CREATE TEMPORARY TABLE _WLINES (
+    wline INTEGER NOT NULL PRIMARY KEY,
+    locationid INTEGER NOT NULL,
+    msgid INTEGER NOT NULL
+  );
+  CREATE TEMPORARY TABLE _BUILDWARNINGS (
+    warningid INTEGER NOT NULL PRIMARY KEY,
+    id INTEGER NOT NULL,
+    wcount INTEGER NOT NULL DEFAULT 1
+  );
+  CREATE TEMPORARY TABLE _BUILDWARNINGLINES (
+    id INTEGER NOT NULL,
+    wline TINYINT NOT NULL,
+    CONSTRAINT PRIMARY KEY (id, wline),
+    locationid INTEGER NOT NULL
+  );
+END
+/;
+
+DROP FUNCTION IF EXISTS getwarningid;
+DROP PROCEDURE IF EXISTS getwarningid;
+CREATE PROCEDURE getwarningid(OUT _warningid INTEGER)
+MODIFIES SQL DATA
+BEGIN
+  -- Given the _WLINES temporary table, figure out the warningid, perhaps
+  -- by inserting a new one.
+  DECLARE haslock INTEGER;
+
+  REPEAT
+    SET @haslock = GET_LOCK('warningdb_warningid', 5);
+  UNTIL @haslock
+  END REPEAT;
+
+  SET _warningid = (
+    SELECT warningid
+    FROM warnings
+    WHERE linecount = (
+      SELECT COUNT(*)
+      FROM warninglines, _WLINES
+      WHERE warnings.warningid = warninglines.warningid
+        AND warninglines.wline = _WLINES.wline
+        AND warninglines.locationid = _WLINES.locationid
+        AND warninglines.msgid = _WLINES.msgid
+    )
+  );
+  IF _warningid IS NULL THEN
+    INSERT INTO warnings (linecount, primaryline)
+    VALUES (
+      (SELECT COUNT(*) FROM _WLINES),
+      0
+    );
+    SET _warningid = LAST_INSERT_ID();
+    INSERT INTO warninglines (warningid, wline, locationid, msgid)
+      SELECT _warningid, wline, locationid, msgid FROM _WLINES;
+    COMMIT;
+  END IF;
+
+  SET @haslock = RELEASE_LOCK('warningdb_warningid');
+END
+/;
+
+DROP PROCEDURE IF EXISTS insertbuild;
+CREATE PROCEDURE insertbuild(buildnumber INTEGER, rev VARCHAR(40))
+MODIFIES SQL DATA
+BEGIN
+  -- Takes a temporary table #BUILDDATA and inserts the data into the database.
+  -- Schema of #BUILDDATA should be id, count, ord, file, line, msg,
+  --   blametype, blamewho, blamefile, blamerev, blameline, loc, blameloc, msgid
+
+  DECLARE done INT DEFAULT 0;
+  DECLARE _curid, _curord, _id, _ord, _wcount, _msgid, _warningid INTEGER;
+  DECLARE _blametype VARCHAR(10);
+  DECLARE _file, _blamefile VARCHAR(255);
+  DECLARE _line, _blameline INTEGER;
+  DECLARE _loc, _blameloc INTEGER;
+  DECLARE _blamerev VARCHAR(40);
+  DECLARE _blamewho VARCHAR(400);
+  DECLARE _msg VARCHAR(255);
+
+  DECLARE readlocs CURSOR FOR
+    SELECT id, ord, wcount, file, line, msg, blametype, blamewho, blamefile, blamerev, blameline
+    FROM _BUILDDATA
+    ORDER BY id, ord;
+  DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;
+
+  DELETE FROM _WLINES;
+  DELETE FROM _BUILDWARNINGS;
+  DELETE FROM _BUILDWARNINGLINES;
+
+  OPEN readlocs;
+  readloop: LOOP
+    FETCH readlocs
+      INTO _id, _ord, _wcount, _file, _line, _msg, _blametype, _blamewho, _blamefile, _blamerev, _blameline;
+
+    IF done OR (_curid is not NULL and _curid != _id) THEN
+      CALL getwarningid(_warningid);
+      DELETE FROM _WLINES;
+
+      INSERT INTO _BUILDWARNINGS (warningid, id, wcount)
+        VALUES (_warningid, _curid, _wcount)
+        ON DUPLICATE KEY UPDATE wcount = wcount + _wcount;
+
+      SET _curord = NULL;
+    END IF;
+
+    IF done THEN LEAVE readloop; END IF;
+
+    SET _curid = _id;
+    IF _curord IS NOT NULL AND _curord != _ord - 1 THEN
+      PREPARE throw_err FROM 'CALL incoming_data_error, ord were not sequential';
+      EXECUTE throw_err;
+    END IF;
+
+    SET _curord = _ord;
+
+    IF _blametype IS NULL THEN
+      SET _blametype = 'system';
+    END IF;
+    IF _line IS NULL THEN
+      SET _line = -1;
+    END IF;
+    IF _blamewho IS NULL THEN
+      SET _blamewho = '';
+    END IF;
+    IF _blamefile IS NULL THEN
+      SET _blamefile = SUBSTRING_INDEX(_file, '/', -1);
+    END IF;
+    IF _blamerev IS NULL THEN
+      SET _blamerev = '';
+    END IF;
+    IF _blameline IS NULL THEN
+      SET _blameline = _line;
+    END IF;
+
+    INSERT INTO locations (ltype, lfile, lline, lrev, lwho)
+      VALUES (_blametype, _blamefile, _blameline, _blamerev, _blamewho)
+      ON DUPLICATE KEY UPDATE locationid = LAST_INSERT_ID(locationid);
+    SET _blameloc = LAST_INSERT_ID();
+    COMMIT;
+
+    INSERT INTO locations(ltype, lfile, lline, lrev, lwho)
+      VALUES ('build', _file, _line, '', '')
+      ON DUPLICATE KEY UPDATE locationid = LAST_INSERT_ID(locationid);
+    SET _loc = LAST_INSERT_ID();
+    COMMIT;
+
+    INSERT INTO messages (msg)
+      VALUES (_msg)
+      ON DUPLICATE KEY UPDATE msgid = LAST_INSERT_ID(msgid);
+    SET _msgid = LAST_INSERT_ID();
+    COMMIT;
+
+    INSERT INTO _WLINES (wline, locationid, msgid)
+      VALUES (_curord, _blameloc, _msgid);
+    INSERT INTO _BUILDWARNINGLINES (id, wline, locationid)
+      VALUES (_curid, _curord, _loc);
+  END LOOP;
+
+  START TRANSACTION;
+  INSERT INTO builds (buildnumber, rev) VALUES (buildnumber, rev);
+  INSERT INTO buildwarnings (buildnumber, warningid, wcount)
+    SELECT buildnumber, warningid, wcount FROM _BUILDWARNINGS;
+  INSERT INTO buildwarninglines (buildnumber, warningid, wline, locationid)
+    SELECT buildnumber, warningid, wline, locationid
+    FROM _BUILDWARNINGLINES, _BUILDWARNINGS
+    WHERE _BUILDWARNINGLINES.id = _BUILDWARNINGS.id;
+  COMMIT;
+END
+/;
--- a/warning-ui/new.html
+++ b/warning-ui/new.html
@@ -6,14 +6,14 @@
   <head>
     <title>New Warnings</title>
   </head>
   <body class="build">
     <h1>New Warnings</h1>
     <p><em>Between build ${start} and ${end}</em></p>
 
     <ul>
-      <li py:for="signature, file, lineno, msg in newwarnings">
-        <a href="${genlink('warning', signature=signature)}">${file}:${lineno} - ${msg}</a>
+      <li py:for="id, file, lineno, msg in newwarnings">
+        <a href="${genlink('warning', id=id)}">${file}:${lineno} - ${msg}</a>
       </li>
     </ul>
   </body>
 </html>
--- a/warning-ui/search.html
+++ b/warning-ui/search.html
@@ -14,18 +14,18 @@
       <li py:if="path != ''">Path LIKE '${path}'</li>
     </ul>
 
     <h2>Results</h2>
 
     <p py:if="len(results) == 0">No warnings found!</p>
 
     <ul py:if="len(results)">
-      <li py:for="signature, file, lineno, msg in results">
-        <a href="${genlink('warning', signature=signature)}">${file}:${lineno} - ${msg}</a>
+      <li py:for="id, file, lineno, msg in results">
+        <a href="${genlink('warning', id=id)}">${file}:${lineno} - ${msg}</a>
       </li>
     </ul>
 
     <h2>Search again:</h2>
     ${searchform(id, user, path, msg)}
 
   </body>
 </html>
--- a/warning-ui/ui.py
+++ b/warning-ui/ui.py
@@ -1,34 +1,32 @@
-import cherrypy, os, sys, re, sqlite3
+import cherrypy, os, sys, re, MySQLdb
 from genshi.template import TemplateLoader
 
 class Root(object):
     def __init__(self):
         self.loader = TemplateLoader(cherrypy.config['tools.staticdir.root'],
                                      auto_reload=True)
 
     def dbcursor(self):
-        return sqlite3.connect(cherrypy.request.app.config['database']['path']).cursor()
+        return MySQLdb.connect(**cherrypy.request.app.config['database']).cursor()
 
     def render(self, tmpl, **kwargs):
         cherrypy.response.headers['Content-Type'] = 'application/xhtml+xml'
         tmpl = self.loader.load(tmpl)
         return tmpl.generate(**kwargs).render('xhtml', doctype='xhtml')
 
     @cherrypy.expose
     def index(self):
+        cur = self.dbcursor()
         try:
-            cur = self.dbcursor()
-            cur.execute('''SELECT builds.buildnumber, builds.rev, uwarnings.ucount
-                           FROM builds LEFT OUTER JOIN
-                             (SELECT buildnumber, count(*) AS ucount
-                              FROM warnings
-                              GROUP BY buildnumber) AS uwarnings
-                           ON builds.buildnumber = uwarnings.buildnumber
+            cur.execute('''SELECT builds.buildnumber, builds.rev, count(*)
+                           FROM builds, buildwarnings
+                           WHERE builds.buildnumber = buildwarnings.buildnumber
+                           GROUP BY builds.buildnumber
                            ORDER BY builds.buildnumber DESC LIMIT 100''')
             return self.render('index.html', builds=cur.fetchall())
         finally:
             cur.close()
 
     @cherrypy.expose
     def robots_txt(self):
         cherrypy.response.headers['Content-Type'] = 'text/plain'
@@ -41,85 +39,75 @@ Disallow: /"""
         span = int(span)
 
         cur = self.dbcursor()
         if end == -1:
             cur.execute('''SELECT max(builds.buildnumber)
                            FROM builds''')
             end, = cur.fetchone()
 
-        cur.execute('''SELECT min(builds.buildnumber)
-                       FROM builds
-                       WHERE builds.buildnumber > ?''', (end - span,))
+        cur.execute('''SELECT MIN(buildnumber)
+                       FROM (
+                         SELECT buildnumber FROM builds
+                         WHERE buildnumber <= %s
+                         ORDER BY buildnumber DESC LIMIT %s
+                       ) AS b''', (end, span))
 
         start, = cur.fetchone()
 
-        cur.execute('''SELECT warnings.signature, file, lineno, msg
-                       FROM warnings, wlines
+        cur.execute('''SELECT warningid, lfile, lline, msg
+                       FROM v_buildwarnings
                        WHERE
-                         warnings.buildnumber = wlines.buildnumber AND
-                         warnings.signature = wlines.signature AND
-                         wlines.wline = 0 AND
-                         warnings.buildnumber = ? AND
-                         NOT EXISTS (SELECT *
-                                     FROM warnings as oldwarnings
-                                     WHERE
-                                       oldwarnings.signature = warnings.signature AND
-                                       oldwarnings.buildnumber = ?)
-                       ORDER BY file''',
-                    (end, start))
+                         buildnumber = %s
+                         AND NOT EXISTS (
+                           SELECT * FROM buildwarnings AS oldwarnings
+                           WHERE oldwarnings.warningid = v_buildwarnings.warningid
+                           AND oldwarnings.buildnumber = %s)''',
+                         (end, start))
         newwarnings = cur.fetchall()
         cur.close()
         return self.render('new.html',
                            start=start,
                            end=end,
                            newwarnings=newwarnings)
-        
+
     @cherrypy.expose
     def search(self, id, user="", path="", msg=""):
         id = int(id)
         user = user.strip()
         path = path.strip()
         msg = msg.strip()
-        
+
         bindp = [id]
+        q = '''SELECT warningid, lfile, lline, msg
+               FROM v_buildwarnings AS bw
+               WHERE bw.buildnumber = %s '''
 
-        subclauses = []
         if user != '':
-            subclauses.append('AND wl2.blamewho LIKE ?')
+            q += '''AND EXISTS (
+                      SELECT * FROM v_warninglines AS vw
+                      WHERE vw.lwho LIKE %s
+                        AND vw.warningid = bw.warningid
+                        AND vw.wline = bw.primaryline
+                    ) '''
             bindp.append(user)
 
         if path != '':
-            subclauses.append('AND wl2.file LIKE ?')
+            q += '''AND bw.lline LIKE %s '''
             bindp.append(path)
 
         if msg != '':
-            subclauses.append('AND wl2.msg LIKE ?')
+            q += '''AND bw.msg LIKE %s '''
             bindp.append(msg)
 
-        if len(subclauses):
-            clause = '''AND EXISTS (SELECT *
-                                    FROM wlines AS wl2
-                                    WHERE
-                                      wl2.buildnumber = warnings.buildnumber AND
-                                      wl2.signature = warnings.signature %s)''' % ' '.join(subclauses)
-        else:
-            clause = ''
+        q += 'ORDER BY lfile, lline'
 
         try:
             cur = self.dbcursor()
-            q = '''SELECT warnings.signature, file, lineno, msg
-                   FROM warnings, wlines
-                   WHERE
-                     warnings.buildnumber = wlines.buildnumber AND
-                     warnings.signature = wlines.signature AND
-                     wlines.wline = 0 AND
-                     warnings.buildnumber = ? %s''' % (clause, )
-            print "Query: %s\nBind: %s" % (q, bindp)
-
+            print "sql: %s bind: %r" % (q, bindp)
             cur.execute(q, bindp)
 
             return self.render('search.html',
                                id=id,
                                user=user,
                                path=path,
                                msg=msg,
                                results=cur.fetchall())
@@ -128,114 +116,115 @@ Disallow: /"""
 
     @cherrypy.expose
     def build(self, id):
         id = int(id)
         try:
             cur = self.dbcursor()
             cur.execute('''SELECT rev
                            FROM builds
-                           WHERE buildnumber = ?''', (id,))
+                           WHERE buildnumber = %s''', (id,))
             rev, = cur.fetchone()
 
-            cur.execute('''SELECT count(*) as ucount, sum(count) AS tcount
-                           FROM warnings
-                           WHERE buildnumber = ?''', (id,))
+            cur.execute('''SELECT count(*) as ucount, sum(wcount) as wcount
+                           FROM buildwarnings
+                           WHERE buildnumber = %s''', (id,))
             unique, total = cur.fetchone()
 
             cur.execute('''SELECT buildnumber, rev
                            FROM builds
                            WHERE
-                             buildnumber < ?
+                             buildnumber < %s
                            ORDER BY buildnumber DESC
                            LIMIT 1''', (id,))
             prev = cur.fetchone()
             if prev is None:
                 previd = None
                 prevrev = None
                 newwarnings = None
                 fixedwarnings = None
             else:
                 previd, prevrev = prev
 
-                cur.execute('''SELECT warnings.signature, file, lineno, msg
-                               FROM warnings, wlines
+                cur.execute('''SELECT warningid, lfile, lline, msg
+                               FROM v_buildwarnings
                                WHERE
-                                 warnings.buildnumber = wlines.buildnumber AND
-                                 warnings.signature = wlines.signature AND
-                                 wlines.wline = 0 AND
-                                 warnings.buildnumber = ? AND
-                                 NOT EXISTS (SELECT *
-                                             FROM warnings AS oldwarnings
-                                             WHERE oldwarnings.signature = warnings.signature AND
-                                             oldwarnings.buildnumber = ?)''', (id, previd))
+                                 buildnumber = %s
+                                 AND NOT EXISTS (
+                                   SELECT *
+                                   FROM buildwarnings AS oldwarnings
+                                   WHERE oldwarnings.warningid = v_buildwarnings.warningid
+                                     AND oldwarnings.buildnumber = %s)''',
+                            (id, previd))
                 newwarnings = cur.fetchall()
 
-                cur.execute('''SELECT warnings.signature, file, lineno, msg
-                               FROM warnings, wlines
+                cur.execute('''SELECT warningid, lfile, lline, msg
+                               FROM v_buildwarnings
                                WHERE
-                                 warnings.buildnumber = wlines.buildnumber AND
-                                 warnings.signature = wlines.signature AND
-                                 wlines.wline = 0 AND
-                                 warnings.buildnumber = ? AND
-                                 NOT EXISTS (SELECT *
-                                             FROM warnings AS oldwarnings
-                                             WHERE oldwarnings.signature = warnings.signature AND
-                                             oldwarnings.buildnumber = ?)''', (previd, id))
+                                 buildnumber = %s
+                                 AND NOT EXISTS (
+                                   SELECT *
+                                   FROM buildwarnings AS oldwarnings
+                                   WHERE oldwarnings.warningid = v_buildwarnings.warningid
+                                     AND oldwarnings.buildnumber = %s)''',
+                            (previd, id))
                 fixedwarnings = cur.fetchall()
-                
+
             return self.render('build.html',
                                rev=rev,
                                id=id,
                                unique=unique,
                                total=total,
                                previd=previd,
                                prevrev=prevrev,
                                newwarnings=newwarnings,
                                fixedwarnings=fixedwarnings)
         finally:
             cur.close()
 
     @cherrypy.expose
-    def warning(self, signature):
+    def warning(self, id):
         try:
             cur = self.dbcursor()
-            cur.execute('''SELECT buildnumber, rev
-                           FROM builds
-                           WHERE
-                             EXISTS (SELECT *
-                                     FROM warnings
-                                     WHERE
-                                       warnings.buildnumber = builds.buildnumber AND
-                                       warnings.signature = ?)
-                           ORDER BY buildnumber ASC LIMIT 1''', (signature,))
-
-            firstid, firstrev = cur.fetchone()
-            
-            cur.execute('''SELECT buildnumber, rev
-                           FROM builds
+            cur.execute('''SELECT wmin.buildnumber, rev
+                           FROM builds,
+                             (
+                                SELECT MIN(buildnumber) AS buildnumber
+                                FROM buildwarnings
+                                WHERE warningid = %s
+                             ) AS wmin
                            WHERE
-                             NOT EXISTS (SELECT *
-                                         FROM warnings
-                                         WHERE
-                                           warnings.buildnumber = builds.buildnumber AND
-                                           warnings.signature = ?) AND
-                             buildnumber > ?
-                           ORDER BY buildnumber ASC LIMIT 1''', (signature, firstid))
+                             builds.buildnumber = wmin.buildnumber''', (id,))
+            firstid, firstrev = cur.fetchone()
+
+            cur.execute('''SELECT wmax.buildnumber, rev
+                           FROM builds,
+                             (
+                                SELECT MAX(buildnumber) AS buildnumber
+                                FROM buildwarnings
+                                WHERE warningid = %s
+                             ) AS wmax
+                           WHERE
+                             builds.buildnumber = wmax.buildnumber''', (id,))
+            lastid, lastrev = cur.fetchone()
 
-            lastid, lastrev = cur.fetchone() or (None, None)
+            cur.execute('''SELECT bwl.lfile, bwl.lline,
+                             wl.msg, wl.ltype, wl.lfile, wl.lline, wl.lrev, wl.lwho
+                           FROM v_buildwarninglines AS bwl,
+                             v_warninglines AS wl
+                           WHERE bwl.buildnumber = %s
+                             AND bwl.warningid = %s
+                             AND wl.warningid = bwl.warningid
+                             AND wl.wline = bwl.wline
+                           ORDER BY wl.wline ASC''', (lastid, id))
+            wlines = cur.fetchall()
 
-            cur.execute('''SELECT file, lineno, msg, blametype, blamefile, blamerev, blameline, blamewho
-                           FROM wlines
-                           WHERE
-                             buildnumber = ? AND
-                             signature = ?
-                           ORDER BY wline ASC''', (firstid, signature))
-
-            wlines = cur.fetchall()
+            if lastid == firstid:
+                lastid = None
+                lastrev = None
 
             return self.render('warning.html',
                                firstid=firstid, firstrev=firstrev,
                                lastid=lastid, lastrev=lastrev,
                                wlines=wlines)
         finally:
             cur.close()
 
--- a/warning-ui/warning.html
+++ b/warning-ui/warning.html
@@ -15,17 +15,17 @@
       <a href="${genlink('build', id=firstid)}">${firstrev}</a>
     </div>
     <div class="h2block" py:if="lastid is not None">
       <h2>Fixed in:</h2>
       <a href="${genlink('build', id=lastid)}">${lastrev}</a>
     </div>
 
     <dl>
-      <py:for each="file, lineno, msg, blametype, blamefile, blamerev, blameline, blamewho in wlines">
+      <py:for each="file, lineno, msg, blametype, blamefile, blameline, blamerev, blamewho in wlines">
         <dt>
           <a py:strip="blametype is None"
              href="http://hg.mozilla.org/mozilla-central/file/${firstrev}/${file}#l${lineno}">
             ${file}:${lineno}</a>:
           ${msg}
         </dt>
         <dd py:if="blametype is not None">
           <py:choose test="blametype">
--- a/warningstep.py
+++ b/warningstep.py
@@ -1,186 +1,92 @@
 from buildbot.process.buildstep import SUCCESS, FAILURE
 from buildbot.steps.shell import ShellCommand
-import sqlite3, os.path
+import MySQLdb, os.path, sys
 
 class WarningParserStep(ShellCommand):
     """
     A subclass of shellcommand used for running warning-parser.py.
 
     This parses the output and saves interesting information to the
     master-side sqlite database. It produces output about "new" warnings, etc.
 
     If the dbfile does not exist, it will be created the first time it is needed.
     """
 
-    _createSchema = """
-    CREATE TABLE builds (
-      buildnumber INTEGER NOT NULL PRIMARY KEY,
-      rev TEXT NOT NULL
-    );
-
-    CREATE TABLE warnings (
-      buildnumber INTEGER NOT NULL CONSTRAINT warnings_buildnumer_fkey REFERENCES builds (buildnumber),
-      signature TEXT NOT NULL,
-      count INTEGER NOT NULL,
-      CONSTRAINT warnings_pkey PRIMARY KEY (buildnumber, signature)
-    );
-
-    CREATE TABLE wlines (
-      buildnumber INTEGER NOT NULL,
-      signature TEXT NOT NULL,
-      wline INTEGER NOT NULL,
-      file TEXT NOT NULL,
-      lineno INTEGER,
-      msg TEXT NOT NULL,
-      blametype TEXT,
-      blamefile TEXT,
-      blamerev TEXT,
-      blameline INTEGER,
-      blamewho TEXT,
-      CONSTRAINT wlines_refwarning_fkey FOREIGN KEY (buildnumber, signature) REFERENCES warnings (buildnumber, signature),
-      CONSTRAINT wlines_pkey PRIMARY KEY (buildnumber, signature, wline)
-    );
-    """
-
-    def __init__(self, dbfile, **kwargs):
+    def __init__(self, connectargs, **kwargs):
         ShellCommand.__init__(self, **kwargs)
 
-        self.dbfile = dbfile
-        self.addFactoryArguments(dbfile=dbfile)
+        self.connectargs = connectargs
+        self.addFactoryArguments(connectargs=connectargs)
 
     def createSummary(self, log):
         stepresults = self.step_status.getBuild().getSteps()
         for sr in stepresults:
             result, str = sr.getResults()
             if result == FAILURE:
                 self.addCompleteLog('skipped', 'Skipped analyzing warnings because a prior step failed.')
                 return
 
-        createSchema = not os.path.exists(self.dbfile)
-
-        db = sqlite3.connect(self.dbfile)
-        cur = db.cursor()
-
-        if createSchema:
-            cur.executescript(self._createSchema)
-
-        cur.execute('PRAGMA synchronous = NORMAL');
-
-        buildnumber = self.getProperty('buildnumber')
+        comparison = self.parseLog(log.getText(), self.getProperty('buildnumber'), self.getProperty('got_revision'))
+        self.addCompleteLog('comparison', '\n'.join(comparison))
 
-        cur.execute('INSERT INTO builds (buildnumber, rev) VALUES ( ?, ? )',
-                    (buildnumber, self.getProperty('got_revision')))
+    def parseLog(self, logdata, buildnumber, revision):
+        db = MySQLdb.connect(**self.connectargs)
+        dbcur = db.cursor()
+        dbcur.arraysize = 200
 
-        def lineSignature(line):
-            """
-            Given a line tuple, return a string signature for the line. This signature is based
-            on the blame location if available, else the reported location.
-            """
-            file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline = line
-
-            if blametype is None:
-                return "%s:%s:%s" % (file, lineno, msg[:30])
-
-            return "%s:%s:%s:%s:%s" % (blametype, blamepath, blamerev[:12], blameline, msg[:30])
+        dbcur.callproc('setuptables')
 
-        def processWarning(lines):
-            """
-            process a warning. A warning is a list of lines. Each line is a tuple of
-            (file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline)
-            """
-
-            signature = "\t".join( (lineSignature(line) for line in lines) )
-
-            cur.execute('''UPDATE warnings
-                           SET count = count + 1
-                           WHERE buildnumber = ? AND signature = ?''', (buildnumber, signature))
-            if cur.rowcount == 0:
-                cur.execute('INSERT INTO warnings (buildnumber, signature, count) VALUES (?, ?, ?)', (buildnumber, signature, 1))
+        def getmany():
+            i = 0
+            for line in logdata.split('\n'):
+                if line.startswith('WARN-DB: '):
+                    i += 1
+                    d = eval(line[9:].strip())
+                    print repr(d)
+                    yield d
 
-                for i in xrange(0, len(lines)):
-                    file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline = lines[i]
-                    cur.execute('''INSERT INTO wlines (buildnumber, signature, wline, file, lineno, msg, blametype, blamefile, blamerev, blameline, blamewho)
-                                   VALUES             (?,           ?,         ?,     ?,    ?,      ?,   ?,         ?,         ?,        ?,         ?)''',
-                                (buildnumber, signature, i, file, lineno, msg, blametype, blamepath, blamerev, blameline, blamewho))
-
+        dbcur.executemany('''INSERT INTO _BUILDDATA (id, ord, file, line, msg, blametype, blamewho, blamefile, blamerev, blameline) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)''',
+                          [eval(line[9:].strip())
+                           for line in logdata.split('\n')
+                           if line.startswith('WARN-DB: ')])
 
-        curwarning = None
-        for line in log.getText().split("\n"):
-            if line.startswith('WARN-DB: '):
-                id, ord, file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline = eval(line[9:].strip())
-
-                if ord == 0:
-                    if curwarning is not None:
-                        processWarning(curwarning)
-                    curwarning = []
-
-                curwarning.append( (file, lineno, msg, blametype, blamewho, blamepath, blamerev, blameline) )
-
-        if curwarning is not None:
-            processWarning(curwarning)
-
+        dbcur.callproc('insertbuild', (buildnumber, revision))
         db.commit()
 
         # A list of line to stick in the comparison log
         comparison = []
 
-        cur.execute('SELECT sum(count) FROM warnings WHERE buildnumber = ?', (buildnumber,))
-        wcount, = cur.fetchone()
-        comparison.append('TinderboxPrint:<a href="http://office.smedbergs.us:8080/build?id=%i">warn:%i</a>' % (buildnumber, wcount))
+        dbcur.execute('''SELECT COUNT(*) as unqiue, SUM(wcount) AS total
+                       FROM buildwarnings
+                       WHERE buildnumber = %s''', (buildnumber,))
+        unique, total = dbcur.fetchone()
+        comparison.append('TinderboxPrint:<a href="http://office.smedbergs.us:8080/build?id=%i">warn:%i</a>' % (buildnumber, unique))
+        comparison.append('TinderboxPrint:(%s total)' % (total))
 
-        cur.execute('SELECT max(buildnumber) FROM builds WHERE buildnumber < ?', (buildnumber,))
-        r = cur.fetchone()
+        dbcur.execute('SELECT MAX(buildnumber) FROM builds WHERE buildnumber < %s', (buildnumber,))
+        r = dbcur.fetchone()
         if r is None:
             comparison.append('no prior build found: skipping comparison')
         else:
             prevbuildnumber, = r
 
-            cur.execute('''SELECT signature
-                           FROM warnings
-                           WHERE buildnumber = ? AND
-                             NOT EXISTS (SELECT *
-                                         FROM warnings AS oldwarnings
-                                         WHERE oldwarnings.signature = warnings.signature AND
-                                           oldwarnings.buildnumber = ?)''', (buildnumber, prevbuildnumber))
-            cur2 = db.cursor()
-
-            wnewcount = 0
-
-            for signature, in cur:
-                wnewcount = wnewcount + 1
-                cur2.execute('''SELECT file, lineno, msg, blametype, blamefile, blamerev, blameline, blamewho
-                                FROM wlines
-                                WHERE buildnumber = ? AND signature = ?
-                                ORDER BY wline''',
-                             (buildnumber, signature))
-                comparison.append('NEW WARNING:')
+            dbcur.execute('''SELECT COUNT(*)
+                             FROM buildwarnings
+                             WHERE buildnumber = %s AND
+                               NOT EXISTS (SELECT *
+                                           FROM buildwarnings AS oldwarnings
+                                           WHERE oldwarnings.warningid = buildwarnings.warningid AND
+                                             oldwarnings.buildnumber = %s)''', (buildnumber, prevbuildnumber))
+            wnewcount, = dbcur.fetchone()
+            if wnewcount > 0:
+                comparison.append('TinderboxPrint:<a href="http://office.smedbergs.us:8080/build?id=%i">warn-new:%i</a>' % (buildnumber, wnewcount))
 
-                for file, lineno, msg, blametype, blamefile, blamerev, blameline, blamewho in cur2:
-                    out = "  %s" % file
-                    if lineno is not None:
-                        out += ":%i" % lineno
-
-                    out += ": %s" % msg
-
-                    if blamewho is not None:
-                        out += ' - Blamed on %s' % blamewho
+        dbcur.close()
+        db.close()
 
-                    if blametype == 'cvs':
-                        out += ' -  http://bonsai.mozilla.org/cvsblame.cgi?file=mozilla/%s&rev=HG_REPO_INITIAL_IMPORT&mark=%s#%s' % (blamefile, blameline, blameline)
-                    elif blametype == 'hg':
-                        out += ' - http://hg.mozilla.org/mozilla-central/annotate/%s/%s#l%i' % (blamerev, blamefile, blameline)
-
-                    comparison.append(out)
-
-        if wnewcount > 0:
-            comparison.append('TinderboxPrint:<a href="http://office.smedbergs.us:8080/build?id=%i">warn-new:%i</a>' % (buildnumber, wnewcount))
-
-        self.addCompleteLog('comparison', '\n'.join(comparison))
-
-        cur.close()
-        db.close()
+        return comparison
 
     def evaluateCommand(self, cmd):
         if cmd.rc != 0:
             return FAILURE
         return SUCCESS