awesomesville, population this commit
authorAndrew Sutherland <asutherland@asutherland.org>
Mon, 26 Oct 2009 03:02:50 -0700
changeset 13 122b184a696894a2675257b0af0d430cadd6eecf
parent 12 ec1b73aa9b359681afc53613565e2deaf11edac9
child 14 c335bd85ec00138dadfd4c374ea1946f8d27fba9
push id2
push userbugmail@asutherland.org
push dateMon, 26 Oct 2009 10:02:58 +0000
awesomesville, population this commit
chrome/content/formatters.js
chrome/content/logaggr.js
chrome/content/logman.js
chrome/content/logui.js
chrome/content/logviewer.css
chrome/content/logviewer.xhtml
chrome/content/logvis.js
chrome/content/processors.js
chrome/modules/gobbler.js
--- a/chrome/content/formatters.js
+++ b/chrome/content/formatters.js
@@ -78,16 +78,19 @@ let LogFormatters = {
   },
 };
 LogFormatters.subtest = LogFormatters.test;
 
 function stringifyThing(obj, conditionalStr) {
   if (conditionalStr == null)
     conditionalStr = "";
 
+  if (obj == null)
+    return conditionalStr + "null";
+
   if (typeof(obj) == "object") {
     if ("type" in obj)
       return stringifyTypedObj(obj, conditionalStr);
   }
 
   return conditionalStr + obj;
 }
 
--- a/chrome/content/logaggr.js
+++ b/chrome/content/logaggr.js
@@ -1,64 +1,180 @@
+
+
 /**
- * Aggregation logic and such.
+ * Processes log messages from log files extracting per-file and per-bucket
+ *  information.  Extensible for the purposes of keeping logic bite-size,
+ *  performance is not a major concern at this time.
+ *
+ * I'm a bit feverish right now, so this might be a bad idea, but the
+ *  LogProcessor registers itself as interested in new log files.  It then
+ *  decorates the log files with a "aggr" object.
  */
-function LogAggr(logFile) {
-  this.logFile = logFile;
-  logFile.resetNewState();
+let LogProcessor = {
+  _init: function LogProcessor__init() {
+    LogManager.registerListener("onNewLogFile", this.onNewLogFile, this);
+    LogManager.registerListener("onLogFileDone", this.onLogFileDone, this);
 
-  this.bucketAggrs = [];
+    LogManager.registerListener("onTick", this._periodicChew, this);
+  },
+
+  _activeLogFiles: [],
 
-  this.curBucket = null;
-  this.curBucketAggr = null;
-  this.curBucketCount = 0;
-}
+  onNewLogFile: function LogProcessor_onNewLogFile(logFile) {
+    logFile.aggr = {
+      all: {
+      },
+      buckets: [],
+      generation: 0,
+      procMeta: {
+        curBucket: null,
+        curBucketAggr: null,
+        curBucketCount: null,
+        seenContexts: {},
+      },
+    };
+    this._activeLogFiles.push(logFile);
+  },
 
-LogAggr.prototype = {
-  bucketAggrs: null,
-  curBucket: null,
-  curBucketAggr: null,
-  curBucketCount: null,
+  onLogFileDone: function LogProcess_onLogFileDone(logFile) {
+    this._activeLogFiles.splice(this._activeLogFiles.indexOf(logFile), 1);
+    this._chew([logFile]);
+  },
 
-  _chewBucket: function(bucket, bucketAggr) {
+  _chewBucket: function LogProcess__chewBucket(logFile, aggr,
+                                               bucket, bucketAggr,
+                                               startFrom) {
+    if (startFrom == null)
+      startFrom = 0;
     let counts = bucketAggr.loggerCounts;
+    let seenContexts = aggr.procMeta.seenContexts;
 
-    for each (let [, msg] in Iterator(bucket)) {
+    for (let iMsg = startFrom; iMsg < bucket.length; iMsg++) {
+      let msg = bucket[iMsg];
+      let context = null;
+
       if (msg.loggerName in counts)
         counts[msg.loggerName]++;
       else
         counts[msg.loggerName] = 1;
+
+      for each (let [, msgObj] in Iterator(msg.messageObjects)) {
+        if ((msgObj == null) || (typeof(msgObj) != "object"))
+          continue;
+
+        // --- Contexts!
+        if ("_isContext" in msgObj) {
+          // -- Just reuse the context meta we already built if we've already
+          //  processed the context.
+          if (msgObj._id in seenContexts) {
+            context = seenContexts[msgObj._id];
+          }
+          // -- special contexts
+          else if ("_specialContext" in msgObj) {
+            // don't mark this as a seen context since it's special
+            let sc = msgObj._specialContext;
+            if (sc in SpecialContextProcessors)
+              SpecialContextProcessors[sc].process(logFile, aggr, bucketAggr,
+                                                   msg, msgObj);
+          }
+          // -- typed contexts
+          else if ("type" in msgObj) {
+            let parentContext;
+            if ("_contextParentId" in msgObj)
+              parentContext = seenContexts[msgObj._contextParentId];
+
+            let typ = msgObj.type;
+            if (typ in TypedContextProcessors)
+              seenContexts[msgObj._id] =
+                TypedContextProcessors[typ].makeContext(
+                  logFile, parentContext, aggr, bucketAggr, msg, msgObj);
+          }
+        }
+        // --- Typed objects
+        else if ("type" in msgObj) {
+          let typ = msgObj.type;
+          if (typ in TypedObjectProcessors)
+            TypedObjectProcessors[typ].process(context, aggr, bucketAggr, msg,
+                                               msgObj);
+        }
+      }
     }
   },
 
-  reset: function() {
+  _chew: function LogProcessor__chew(logFiles) {
+    for each (let [, logFile] in Iterator(logFiles)) {
+      let aggr = logFile.aggr;
+      let procMeta = aggr.procMeta;
+      let didSomething = false;
+
+      if (procMeta.curBucket &&
+          procMeta.curBucket.length != procMeta.curBucketCount) {
+        didSomething = true;
+        this._chewBucket(logFile, aggr,
+                         procMeta.curBucket, procMeta.curBucketAggr,
+                         procMeta.curBucketCount);
+      }
+
+      let newBuckets = logFile.getAndClearNewBuckets();
+      for each (let [iBucket, [bucketName, bucket]] in Iterator(newBuckets)) {
+        didSomething = true;
+        let bucketAggr = {
+          name: bucketName,
+          loggerCounts: {}
+        };
+
+        aggr.buckets.push(bucketAggr);
+
+        this._chewBucket(logFile, aggr, bucket, bucketAggr);
+
+        if (iBucket == newBuckets.length - 1) {
+          procMeta.curBucket = bucket;
+          procMeta.curBucketAggr = bucketAggr;
+          procMeta.curBucketCount = bucket.length;
+        }
+      }
+
+      if (didSomething)
+        aggr.generation++;
+    }
   },
 
-  chew: function() {
-    let didSomething = false;
+  _periodicChew: function() {
+    this._chew(this._activeLogFiles);
+  },
 
-    if (this.curBucket && this.curBucket.length != this.curBucketCount) {
-      didSomething = true;
-      this._chewBucket(this.curBucket, this.curBucketAggr);
-    }
+  /**
+   * Maps a listeningFor identifier to a list of listeners.
+   */
+  _listenersByListeningFor: {},
+
+  notify: function LogManager__notifyListeners(processorName, processorEvent,
+                                               args) {
+    let listeningFor = processorName + "-" + processorEvent;
 
-    let newBuckets = this.logFile.getAndClearNewBuckets();
-    for each (let [iBucket, [bucketName, bucket]] in Iterator(newBuckets)) {
-      didSomething = true;
-      let bucketAggr = {
-        name: bucketName,
-        loggerCounts: {}
-      };
+    if (!(listeningFor in this._listenersByListeningFor))
+      return;
 
-      this.bucketAggrs.push(bucketAggr);
-
-      this._chewBucket(bucket, bucketAggr);
-
-      if (iBucket == newBuckets.length - 1) {
-        this.curBucket = bucket;
-        this.curBucketAggr = bucketAggr;
-        this.curBucketCount = bucket.length;
+    for each (let [, [listener, listenerThis]] in
+              Iterator(this._listenersByListeningFor[listeningFor])) {
+      try {
+        listener.apply(listenerThis, args);
+      }
+      catch (ex) {
+        dump("!!! exception calling listener " + listener + "\n");
+        dump(ex + "\n");
+        dump(ex.stack + "\n\n");
       }
     }
+  },
 
-    return didSomething;
-  }
+  registerListener: function LogManager_registerListener(
+                      processorName, processorEvent, listener, listenerThis) {
+    let listenFor = processorName + "-" + processorEvent;
+    if (!(listenFor in this._listenersByListeningFor))
+      this._listenersByListeningFor[listenFor] = [];
+
+    this._listenersByListeningFor[listenFor].push([listener, listenerThis]);
+  },
+
 };
+LogProcessor._init();
--- a/chrome/content/logman.js
+++ b/chrome/content/logman.js
@@ -1,19 +1,29 @@
 /*
  * Logging domain logic.
  */
 
-function LogFile() {
+/**
+ * Holds all the log messages for a given source (log file, network connection,
+ *  etc.).
+ *
+ * The |LogManager| crams data into us.
+ */
+function LogFile(id) {
+  this.id = id;
   this.knownLoggers = {};
   this._dateBucketsByName = {};
   this._dateBuckets = [];
   this._newBuckets = [];
 }
 LogFile.prototype = {
+  /**
+   * Name identify the log file; probably the name of the unit test source file.
+   */
   name: null,
 
   /**
    * Maps logger names to information about those loggers.
    *
    * Example keys would be "gloda.indexer", "gloda.ns", etc.
    */
   knownLoggers: null,
@@ -40,32 +50,27 @@ LogFile.prototype = {
     return newBuckets;
   },
 
   getBucket: function LogFile_getBucket(bucketName) {
     if (bucketName in this._dateBucketsByName)
       return this._dateBucketsByName[bucketName];
     return null;
   },
-
-  // stopgap measure until we refactor aggregation into the logfile concept.
-  // there does not seem to be much benefit to decoupling aggregation entirely
-  //  given that limit event processing and summarization is desired in all
-  //  cases.
-  resetNewState: function() {
-    this._newBuckets = [[bucket.name, bucket] for each
-                         ([, bucket] in Iterator(this._dateBuckets))];
-  }
 };
 
 /**
  * The log manager is the clearing house for log data.  Log entries are
  *  organized into LogFile instances.
+ *
+ * Only the LogManager is aware of the realities of
  */
 let LogManager = {
+  _nextId: 1,
+
   PORT: 9363,
 
   DATE_BUCKET_SIZE_IN_MS: 100,
 
   _init: function LogManager__init() {
     this.reset();
     this.gobbler = new LogGobbler(this);
     this.gobbler.start(this.PORT);
@@ -88,32 +93,41 @@ let LogManager = {
   _noteNewLogger: function LogManager__noteNewLogger(logFile, loggerName) {
     logFile.knownLoggers[loggerName] = {};
     this._notifyListeners("onNewLogger", [logFile, loggerName]);
   },
 
   logFiles: null,
 
   onNewConnection: function LogManager_onNewConnection() {
-    let logFile = new LogFile();
+    let logFile = new LogFile(this._nextId++);
     this.logFiles.push(logFile);
+
+    this._notifyListeners("onNewLogFile", [logFile]);
     return logFile;
   },
 
+  onClosedConnection: function LogManager_onClosedConnection(logFile) {
+    // actually maybe nothing to do about this?
+    this._notifyListeners("onLogFileDone", [logFile]);
+  },
+
   /**
    * Process received messages.
    */
   onLogMessage: function LogManager_onLogMessage(logFile, msg) {
     // look for the name packet, it should be the first thing we see.
     if (logFile.name == null) {
       if (msg.messageObjects.length &&
-          ("_isSpecialContext" in msg.messageObjects[0])) {
+          msg.messageObjects[0] != null &&
+          (typeof(msg.messageObjects[0]) == "object") &&
+          ("_specialContext" in msg.messageObjects[0])) {
         let testFile = msg.messageObjects[0].testFile[0];
         logFile.name = testFile.substring(testFile.lastIndexOf("/") + 1);
-        this._notifyListeners("onNewLogFile", [logFile]);
+        this._notifyListeners("onLogFileNamed", [logFile]);
       }
     }
 
     if (!(msg.loggerName in logFile.knownLoggers))
       this._noteNewLogger(logFile, msg.loggerName);
 
     // Let us assume time is monotonically increasing.
     let bucketName = msg.time - msg.time % this.DATE_BUCKET_SIZE_IN_MS;
@@ -156,11 +170,28 @@ let LogManager = {
   },
 
   registerListener: function LogManager_registerListener(listenFor, listener,
                                                          listenerThis) {
     if (!(listenFor in this._listenersByListeningFor))
       this._listenersByListeningFor[listenFor] = [];
 
     this._listenersByListeningFor[listenFor].push([listener, listenerThis]);
+  },
+
+  tick: function LogManager_tick() {
+    this._notifyListeners("onTick", []);
   }
 };
 LogManager._init();
+
+// ugly support for periodically processing logs and maybe updating the UI
+setInterval(function() {
+  try {
+    LogManager.tick();
+  }
+  catch (ex) {
+    dump("!!! exception during update tick\n");
+    dump(ex + "\n");
+    dump(ex.stack + "\n\n");
+  }
+}, 1000);
+
--- a/chrome/content/logui.js
+++ b/chrome/content/logui.js
@@ -6,16 +6,17 @@
  * In charge of the various concepts of selection and such.
  */
 let LogUI = {
   _init: function LogUI__init() {
     LogManager.registerListener("onNewLogFile", this.onNewLogFile, this);
   },
   _uiInit: function LogUI__uiInit() {
     $("#data-tabs").tabs();
+    LogManager.registerListener("onTick", LogUI.tick, LogUI);
   },
 
   selectedLogFile: null,
 
   onNewLogFile: function LogUI_onNewLogFile(logFile) {
     this.selectLogFile(logFile);
   },
 
@@ -56,21 +57,27 @@ let LogUI = {
   },
 
   registerListener: function LogManager_registerListener(listenFor, listener,
                                                          listenerThis) {
     if (!(listenFor in this._listenersByListeningFor))
       this._listenersByListeningFor[listenFor] = [];
 
     this._listenersByListeningFor[listenFor].push([listener, listenerThis]);
+  },
+
+  tick: function LogUI_tick() {
+    this._notifyListeners("onTick", []);
   }
 };
 LogUI._init();
 $(LogUI._uiInit);
 
+
+
 /**
  * In charge of the log listing UI which is slaved to the currently selected
  *  bucket.
  */
 let LogList = {
   _init: function LogList__init() {
     LogUI.registerListener("onBucketSelected", this.onBucketSelected, this);
   },
@@ -148,16 +155,21 @@ DetailView._init();
 /**
  * Implements the "Test Files" tab that shows us the various tests we have heard
  *  about.
  */
 let TestFilesList = {
   _init: function TestFilesList__init() {
     LogManager.registerListener("onReset", this.onReset, this);
     LogManager.registerListener("onNewLogFile", this.onNewLogFile, this);
+    LogManager.registerListener("onLogFileNamed", this.onLogFileNamed, this);
+
+    LogManager.registerListener("onLogFileDone", this.onLogFileFinished, this);
+    LogProcessor.registerListener("lifecycle", "finished",
+                                  this.onLogFileFinished, this);
   },
 
   onReset: function TestFilesList_onReset() {
     let root = document.getElementById("test-files");
     while (root.lastChild)
       root.removeChild(root.lastChild);
   },
 
@@ -168,17 +180,118 @@ let TestFilesList = {
       listRoot = root.lastChild;
     }
     else {
       listRoot = document.createElement("ul");
       root.appendChild(listRoot);
     }
 
     let fileNode = document.createElement("li");
-    fileNode.textContent = logFile.name;
+    fileNode.setAttribute("id", "test-file-" + logFile.id);
+    fileNode.textContent = "unknown with id " + logFile.id;
     fileNode.setAttribute("class", "clicky");
     fileNode.onclick = function() {
       LogUI.selectLogFile(logFile);
     };
     listRoot.appendChild(fileNode);
-  }
+  },
+
+  onLogFileNamed: function TestFileList_onLogFileNamed(logFile) {
+    let fileNode = document.getElementById("test-file-" + logFile.id);
+    fileNode.textContent = logFile.name;
+  },
+
+  onLogFileFinished: function TestFilesList_onLogFileFinished(logFile) {
+    let fileNode = document.getElementById("test-file-" + logFile.id);
+
+    let styleClass = (logFile.aggr.failures ? "failure" :
+                      (logFile.aggr.state == "finished") ?
+                        "success" : "warning");
+    fileNode.setAttribute("class", "clicky " + styleClass);
+  },
 };
 TestFilesList._init();
+
+
+/**
+ * Implements the "Test Files" tab that shows us the various tests we have heard
+ *  about.
+ */
+let TestList = {
+  _init: function TestList__init() {
+    LogManager.registerListener("onReset", this.onReset, this);
+    LogUI.registerListener("onLogFileSelected", this.onLogFileSelected, this);
+
+    LogProcessor.registerListener("test", "new",
+                                  this.onNewTest, this);
+    LogProcessor.registerListener("subtest", "new",
+                                  this.onNewTest, this);
+  },
+
+  selectedLogFile: null,
+
+  onReset: function TestFilesList_onReset() {
+    let root = document.getElementById("test-list");
+    while (root.lastChild)
+      root.removeChild(root.lastChild);
+  },
+
+  onLogFileSelected: function TestList_onLogFileSelected(logFile) {
+    this.onReset();
+    this.selectedLogFile = logFile;
+
+    if (!("tests" in logFile.aggr))
+      return;
+
+    for each (let [, test] in Iterator(logFile.aggr.tests)) {
+      this.onNewTest(logFile, null, test);
+    }
+  },
+
+  onNewTest: function TestList_onNewTest(logFile, parent, test) {
+    if (logFile != this.selectedLogFile)
+      return;
+
+    let listRoot;
+    if (!parent) {
+      let root = document.getElementById("test-list");
+      if (root.lastChild) {
+        listRoot = root.lastChild;
+      }
+      else {
+        listRoot = document.createElement("ul");
+        root.appendChild(listRoot);
+      }
+    }
+    else {
+      let parentNode = document.getElementById("test-" + parent.id);
+      if (parentNode.firstChild != parentNode.lastChild) {
+        listRoot = parentNode.lastChild;
+      }
+      else {
+        listRoot = document.createElement("ul");
+        parentNode.appendChild(listRoot);
+      }
+    }
+
+    let label = test.name;
+    if (test.parameter)
+      label += ": " + test.parameter;
+
+    let testNode = document.createElement("li");
+    testNode.setAttribute("id", "test-" + test.id);
+    let spanNode = document.createElement("span");
+    spanNode.textContent = label;
+    spanNode.setAttribute("class", "clicky");
+    spanNode.onclick = function() {
+      LogUI.selectBucket(test.firstSeenInBucket);
+    };
+    testNode.appendChild(spanNode);
+    listRoot.appendChild(testNode);
+
+    if (test.subtests.length) {
+      for each (let [, subtest] in Iterator(test.subtests)) {
+        this.onNewTest(logFile, test, subtest);
+      }
+    }
+  },
+};
+TestList._init();
--- a/chrome/content/logviewer.css
+++ b/chrome/content/logviewer.css
@@ -10,8 +10,20 @@ body {
 
 #bucket-contents {
   overflow: auto;
 }
 
 dt {
   font-weight: bold;
 }
+
+.success {
+  color: green;
+}
+
+.warning {
+  color: brown;
+}
+
+.failure {
+  color: red;
+}
--- a/chrome/content/logviewer.xhtml
+++ b/chrome/content/logviewer.xhtml
@@ -20,16 +20,17 @@
             src="jslibs/jquery/jquery-1.3.2.min.js"></script>
     <script type="text/javascript"
             src="jslibs/jquery/jquery-ui-1.7.2.custom.min.js"></script>
     <!-- protovis -->
     <script src="jslibs/protovis/protovis-d3.1.js" type="application/javascript;version=1.8"/>
     <!-- us -->
     <script src="logman.js" type="application/javascript;version=1.8"/>
     <script src="logaggr.js" type="application/javascript;version=1.8"/>
+    <script src="processors.js" type="application/javascript;version=1.8"/>
     <script src="logui.js" type="application/javascript;version=1.8"/>
     <script src="logvis.js" type="application/javascript;version=1.8"/>
     <script src="formatters.js" type="application/javascript;version=1.8"/>
     <link rel="stylesheet"
       href="chrome://logsploder/content/logviewer.css"
       type="text/css" />
   </head>
   <body bgcolor="#ffffff">
@@ -37,21 +38,23 @@
       <div id="logger-hierarchy-vis" style="float: left;"/>
       <div id="date-bucket-vis" style="float: left;"/>
     </div>
     <div id="data-tabs">
       <ul>
         <li><a href="#bucket-contents">Log Contents</a></li>
         <li><a href="#detail-view">Detail View</a></li>
         <li><a href="#test-files">Test Files</a></li>
+        <li><a href="#test-list">Tests</a></li>
         <li><a href="#controls">Controls</a></li>
       </ul>
       <div id="bucket-contents"/>
       <div id="detail-view"/>
       <div id="test-files"/>
+      <div id="test-list"/>
       <div id="controls">
         <ul>
           <li class="clicky" onclick="LogManager.reset();">
             Reset: Clear Everything
           </li>
         </ul>
       </div>
     </div>
--- a/chrome/content/logvis.js
+++ b/chrome/content/logvis.js
@@ -102,37 +102,29 @@ let LoggerHierarchyVisier = {
   },
 };
 LoggerHierarchyVisier._init();
 
 
 
 let DateBucketVis = {
   _init: function DateBucketVis__init() {
-    this._updateRequired = true;
-    LogManager.registerListener("onReset", this._onReset, this);
+    LogUI.registerListener("onLogFileSelected", this._onLogFileSelected, this);
 
-    LogUI.registerListener("onLogFileSelected", this._onLogFileSelected, this);
+    LogUI.registerListener("onTick", this.updateVis, this);
   },
 
   selectedLogFile: null,
   logAggr: null,
   _onLogFileSelected: function LoggerHierarchyVisier__onLogFileSelected(
                                  logFile) {
-    this.selectedLogFile = logFile;
-    this.logAggr = new LogAggr(logFile);
-    this.buckets = this.logAggr.bucketAggrs;
+    this.logFile = logFile;
     if (this._cellVis)
-      this._cellVis.data(this.buckets);
-    this._updateRequired = true;
-    this.updateVis();
-  },
-
-  _onReset: function DateBucketVis__onReset() {
-    this._updateRequired = true;
+      this._cellVis.data(logFile.aggr.buckets);
+    this.lastVisedGeneration = -1;
     this.updateVis();
   },
 
   WIDTH: 615,
   HEIGHT: 369,
   CELL_WIDTH: 41,
   CELL_HEIGHT: 41,
   _vis: null,
@@ -161,54 +153,39 @@ let DateBucketVis = {
       if (loggerName in bucketAggr.loggerCounts)
         count = bucketAggr.loggerCounts[loggerName];
       return new pv.Color.Hsl(360 * s.index / s.scene.length,
                               Math.min(1.0, count / 10),
                               0.8, 1);
     };
 
     let cell = this._cellVis = vis.add(pv.Panel)
-      .data(this.buckets)
+      .data(this.logFile.aggr.buckets)
       .top(function() Math.floor(this.index / xCount) * CELL_HEIGHT)
       .left(function() (this.index % xCount) * CELL_WIDTH)
       .height(CELL_HEIGHT - 1)
       .width(CELL_WIDTH - 1)
       .event("click", function(d) LogUI.selectBucket(d));
     this._mapvis = cell.add(pv.Bar)
       .extend(this._treemap)
       .fillStyle(function(n) colorize(this, n));
 
   },
   updateVis: function DateBucketVis__updateVis() {
-    if (!this.logAggr)
+    if (!this.logFile)
       return;
 
-    // don't do anything if nothing changed.
-    if (!this.logAggr.chew() && !this._updateRequired)
+    if (this.logFile.aggr.generation == this.lastVisedGeneration)
       return;
 
-    if ((this.buckets.length == 0) && !this._updateRequired)
-      return;
-    this._updateRequired = false;
-
     if (this._vis == null)
       this._makeVis();
     else {
       this._treemap = pv.Layout.treemap(LoggerHierarchyVisier.loggerTree)
         .round(true);
       this._mapvis.extend(this._treemap);
     }
 
     this._vis.render();
   }
 };
 DateBucketVis._init();
 
-setInterval(function() {
-  try {
-    DateBucketVis.updateVis();
-  }
-  catch (ex) {
-    dump("!!! exception updating DateBucketVis\n");
-    dump(ex + "\n");
-    dump(ex.stack + "\n\n");
-  }
-}, 1000);
new file mode 100644
--- /dev/null
+++ b/chrome/content/processors.js
@@ -0,0 +1,59 @@
+let SpecialContextProcessors = {
+  lifecycle: {
+    process: function scp_lifecycle_process(logFile, aggr, bucketAggr,
+                                            msg, msgObj) {
+      if (msgObj._id == "start") {
+        aggr.state = "running";
+      }
+      else if (msgObj._id == "finish") {
+        aggr.state = "finished";
+        LogProcessor.notify("lifecycle", "finished", [logFile]);
+      }
+    }
+  }
+};
+
+let TypedContextProcessors = {
+  test: {
+    makeContext: function tcp_test_makeContext(logFile, parentContext,
+                                               aggr, bucketAggr, msg, msgObj) {
+      if (!("tests" in aggr))
+        aggr.tests = [];
+
+      let test = {
+        id: msgObj._id,
+        name: msgObj.name,
+        parameter: msgObj.parameter,
+        firstSeenInBucket: bucketAggr,
+        subtests: [],
+      };
+
+      if (parentContext) {
+        parentContext.subtests.push(test);
+        LogProcessor.notify("subtest", "new", [logFile, parentContext, test]);
+      }
+      else {
+        aggr.tests.push(test);
+        LogProcessor.notify("test", "new", [logFile, null, test]);
+      }
+
+      return test;
+    }
+  }
+};
+TypedContextProcessors.subtest = TypedContextProcessors.test;
+
+let TypedObjectProcessors = {
+  failure: {
+    process: function top_failure_process(context, aggr, bucketAggr, msg, msgObj) {
+      if (aggr.failures)
+        aggr.failures++;
+      else
+        aggr.failures = 1;
+      if (bucketAggr.failures)
+        bucketAggr.failures++;
+      else
+        bucketAggr.failures = 1;
+    }
+  }
+};
--- a/chrome/modules/gobbler.js
+++ b/chrome/modules/gobbler.js
@@ -125,53 +125,57 @@ Gobbler.prototype = {
   onStopListening: function Gobbler_onStopListening(aServer, aStatus) {
     dump("onStopListening\n");
   }
 };
 
 function GobblerConnection(aGobbler, aInputStream) {
   this._gobbler = aGobbler;
   this._inputStream = aInputStream;
-  this._scriptableInputStream = Cc["@mozilla.org/scriptableinputstream;1"]
-                                  .createInstance(Ci.nsIScriptableInputStream);
-  this._scriptableInputStream.init(this._inputStream);
+
+  this._uniInputStream = Cc["@mozilla.org/intl/converter-input-stream;1"]
+                           .createInstance(Ci.nsIConverterInputStream);
+  this._uniInputStream.init(this._inputStream, "UTF-8", 0,
+      Ci.nsIConverterInputStream.DEFAULT_REPLACEMENT_CHARACTER);
 
   this._inputStream.asyncWait(this, 0, 0, this._gobbler._mainThread);
 
   this._data = "";
 }
 GobblerConnection.prototype = {
   onInputStreamReady: function (aInputStream) {
     try {
-      this._data += this._scriptableInputStream.read(
-          this._scriptableInputStream.available());
+      let ostr = {};
+      this._uniInputStream.readString(this._inputStream.available(), ostr);
+      this._data += ostr.value;
     }
     catch (ex) {
+      dump("killing connection cause: " + ex + "\n" + ex.stack + "\n\n");
       this.close();
       return;
     }
 
     let idxNewLine;
     while ((idxNewline = this._data.indexOf("\r\n")) != -1) {
       let line = this._data.substring(0, idxNewline);
       this._data = this._data.substring(idxNewline+2);
       this.processLine(line);
     }
 
     this._inputStream.asyncWait(this, 0, 0, this._gobbler._mainThread);
   },
 
   close: function () {
     try {
-      this._scriptableInputStream.close();
+      this._uniInputStream.close();
       this._inputStream.close();
     }
     catch (ex) {
     }
-    this._scriptableInputStream = null;
+    this._uniInputStream = null;
     this._inputStream = null;
   }
 };
 
 /**
  * Specialized log processing server; which is to say, we know how to create
  *
  */
@@ -191,9 +195,13 @@ function LogGobblerConnection() {
   this.handle = this.listener.onNewConnection();
 }
 LogGobblerConnection.prototype = {
   __proto__: GobblerConnection.prototype,
   processLine: function LogGobblerConnection_processLine(aLine) {
     let message = this._gobbler._json.decode(aLine);
     this.listener.onLogMessage(this.handle, message);
   },
+  close: function LogGobblerConnection_close() {
+    this.__proto__.__proto__.close.call(this);
+    this.listener.onClosedConnection(this.handle);
+  }
 };