b=390907, add ability for pageloader to perform self-timed tests, r=shaver
authorvladimir@pobox.com
Mon, 06 Aug 2007 13:59:05 -0700
changeset 4322 dbb9fc57f0363121e485eed396931e8030ba6f04
parent 4321 815709d80775c28751930102f4ebdfd88e3b71a6
child 4323 2a4062a68b7607b659a3fa2a0c3c593b4ba16784
push idunknown
push userunknown
push dateunknown
reviewersshaver
bugs390907
milestone1.9a8pre
b=390907, add ability for pageloader to perform self-timed tests, r=shaver
layout/tools/pageloader/README
layout/tools/pageloader/pageloader.js
layout/tools/pageloader/report.js
layout/tools/pageloader/tp-cmdline.js
--- a/layout/tools/pageloader/README
+++ b/layout/tools/pageloader/README
@@ -2,21 +2,49 @@ Pageload Test Component
 =======================
 
 Usage:
 
   ./firefox -tp file:///path/to/manifest.txt [-tpargs...]
 
 See ./firefox -help for other arguments.
 
-The manifest file should contain a list of URLs or URL fragments, one
-per line.  Empty lines or lines starting with # are ignored.  If URL
-fragments are specified, then -tpprefix must be used to give a prefix
-to prepend to each line in the manifest to turn it into a complete
-URL.
+
+Manifest file format
+====================
+
+Comments in the manifest file start with a #.  Each line may be:
+
+* a URL (absolute or relative to the manifest)
+
+This URL is added to the list of tests.
+
+* one or more flags, followed by whitespace, followed by a URL
+
+The only flag supported currently is '%', which indicates that
+a test will do its own timing.  (See Self-timing Tests below.)
+
+* "include" followed by whitespace, followed by a URL
+
+Parse the given manifest file.
+
+Self-timing Tests
+=================
+
+Most timing tests are interested in timing how long it takes the page
+to load; that is, from the start of page loading until the 'load'
+event is dispatched.  By default, this is what the pageloader will
+time.  However, if a test URL has the % flag, the test is expected to
+report its own timing.  For this purpose, the pageloader will provide
+a function named "tpRecordTime" in the test's global object that it
+should call once it has performed whatever timing it wants to do.
+The given value will be used as the timing result for this test.
+
+Output format
+=============
 
 The result is a dump to stdout via dump() --
 browser.dom.window.dump.enabled must be set to true in the profile.  A
 number of output formats can be specified via the -tpformat command
 line option, currently 'js', 'text', and 'tinderbox' are supported.
 
 Sample 'js' format output:
 
--- a/layout/tools/pageloader/pageloader.js
+++ b/layout/tools/pageloader/pageloader.js
@@ -42,63 +42,64 @@ const Ci = Components.interfaces;
 
 var NUM_CYCLES = 5;
 
 var pageFilterRegexp = null;
 var reportFormat = "js";
 var useBrowser = true;
 var winWidth = 1024;
 var winHeight = 768;
-var urlPrefix = null;
 
 var doRenderTest = false;
 
 var pages;
 var pageIndex;
-var results;
 var start_time;
 var cycle;
 var report;
 var renderReport;
 var running = false;
 
 var content;
 
+var TEST_DOES_OWN_TIMING = 1;
+
 var browserWindow = null;
 
+// the io service
+var gIOS = null;
+
 function plInit() {
   if (running) {
     return;
   }
   running = true;
 
   cycle = 0;
-  results = {};
 
   try {
     var args = window.arguments[0].wrappedJSObject;
 
     var manifestURI = args.manifest;
     var startIndex = 0;
     var endIndex = -1;
     if (args.startIndex) startIndex = parseInt(args.startIndex);
     if (args.endIndex) endIndex = parseInt(args.endIndex);
     if (args.numCycles) NUM_CYCLES = parseInt(args.numCycles);
     if (args.format) reportFormat = args.format;
     if (args.width) winWidth = parseInt(args.width);
     if (args.height) winHeight = parseInt(args.height);
     if (args.filter) pageFilterRegexp = new RegExp(args.filter);
-    if (args.prefix) urlPrefix = args.prefix;
     doRenderTest = args.doRender;
 
-    var ios = Cc["@mozilla.org/network/io-service;1"]
+    gIOS = Cc["@mozilla.org/network/io-service;1"]
       .getService(Ci.nsIIOService);
     if (args.offline)
-      ios.offline = true;
-    var fileURI = ios.newURI(manifestURI, null, null);
+      gIOS.offline = true;
+    var fileURI = gIOS.newURI(manifestURI, null, null);
     pages = plLoadURLsFromURI(fileURI);
 
     if (!pages) {
       dumpLine('tp: could not load URLs, quitting');
       plStop(true);
     }
 
     if (pages.length == 0) {
@@ -111,28 +112,29 @@ function plInit() {
     if (endIndex == -1 || endIndex >= pages.length)
       endIndex = pages.length-1;
     if (startIndex > endIndex) {
       dumpLine("tp: error: startIndex >= endIndex");
       plStop(true);
     }
 
     pages = pages.slice(startIndex,endIndex+1);
-    report = new Report(pages);
+    var pageUrls = pages.map(function(p) { return p.url.spec.toString(); });
+    report = new Report(pageUrls);
 
     if (doRenderTest)
-      renderReport = new Report(pages);
+      renderReport = new Report(pageUrls);
 
     pageIndex = 0;
 
     if (args.useBrowserChrome) {
       var wwatch = Cc["@mozilla.org/embedcomp/window-watcher;1"]
-	.getService(Ci.nsIWindowWatcher);
+        .getService(Ci.nsIWindowWatcher);
       var blank = Cc["@mozilla.org/supports-string;1"]
-	.createInstance(Ci.nsISupportsString);
+        .createInstance(Ci.nsISupportsString);
       blank.data = "about:blank";
       browserWindow = wwatch.openWindow
         (null, "chrome://browser/content/", "_blank",
          "chrome,dialog=no,width=" + winWidth + ",height=" + winHeight, blank);
 
       // get our window out of the way
       window.resizeTo(10,10);
 
@@ -153,148 +155,242 @@ function plInit() {
                    }, 500);
       };
 
       browserWindow.addEventListener('load', browserLoadFunc, true);
     } else {
       window.resizeTo(winWidth, winHeight);
 
       content = document.getElementById('contentPageloader');
-      content.addEventListener('load', plLoadHandler, true);
 
       setTimeout(plLoadPage, 0);
     }
   } catch(e) {
     dumpLine(e);
     plStop(true);
   }
 }
 
+function plPageFlags() {
+  return pages[pageIndex].flags;
+}
+
+// load the current page, start timing
+var removeLastAddedListener = null;
 function plLoadPage() {
+  var pageName = pages[pageIndex].url.spec;
+
+  if (removeLastAddedListener)
+    removeLastAddedListener();
+
+  if (plPageFlags() & TEST_DOES_OWN_TIMING) {
+    // if the page does its own timing, use a capturig handler
+    // to make sure that we can set up the function for content to call
+    content.addEventListener('load', plLoadHandlerCapturing, true);
+    removeLastAddedListener = function() {
+      content.removeEventListener('load', plLoadHandlerCapturing, true);
+    };
+  } else {
+    // if the page doesn't do its own timing, use a bubbling handler
+    // to make sure that we're called after the page's own onload() handling
+
+    // XXX we use a capturing event here too -- load events don't bubble up
+    // to the <browser> element.  See bug 390263.
+    content.addEventListener('load', plLoadHandler, true);
+    removeLastAddedListener = function() {
+      content.removeEventListener('load', plLoadHandler, true);
+    };
+  }
+
   start_time = Date.now();
-  content.loadURI(pages[pageIndex]);
+  content.loadURI(pageName);
 }
 
+function plNextPage() {
+  if (pageIndex < pages.length-1) {
+    pageIndex++;
+
+    setTimeout(plLoadPage, 0);
+  } else {
+    plStop(false);
+  }
+}
+
+function plRecordTime(time) {
+  var pageName = pages[pageIndex].url.spec;
+  report.recordTime(pageIndex, time);
+}
+
+function plLoadHandlerCapturing(evt) {
+  // make sure we pick up the right load event
+  if (evt.type != 'load' ||
+      (!evt.originalTarget instanceof Ci.nsIDOMHTMLDocument ||
+       evt.originalTarget.defaultView.frameElement))
+      return;
+
+  if (!(plPageFlags() & TEST_DOES_OWN_TIMING)) {
+    dumpLine("tp: Capturing onload handler used with page that doesn't do its own timing?");
+    plStop(true);
+  }
+
+  // set up the function for content to call
+  content.contentWindow.wrappedJSObject.tpRecordTime = function (time) {
+    plRecordTime(time);
+    setTimeout(plNextPage, 0);
+  };
+}
+
+// the onload handler
 function plLoadHandler(evt) {
   // make sure we pick up the right load event
   if (evt.type != 'load' ||
       (!evt.originalTarget instanceof Ci.nsIDOMHTMLDocument ||
        evt.originalTarget.defaultView.frameElement))
       return;
 
   var end_time = Date.now();
   var time = (end_time - start_time);
 
-  var pageName = pages[pageIndex];
+  // does this page want to do its own timing?
+  // if so, we shouldn't be here
+  if (plPageFlags() & TEST_DOES_OWN_TIMING) {
+    dumpLine("tp: Bubbling onload handler used with page that does its own timing?");
+    plStop(true);
+  }
 
-  results[pageName] = time;
-  report.recordTime(pageIndex, time);
+  plRecordTime(time);
 
   if (doRenderTest)
     runRenderTest();
 
-  if (pageIndex < pages.length-1) {
-    pageIndex++;
-    setTimeout(plLoadPage, 0);
-  } else {
-    plStop(false);
-  }
+  plNextPage();
 }
 
 function runRenderTest() {
-  const redrawsPerSample = 5;
-  const renderCycles = 10;
+  const redrawsPerSample = 500;
 
   if (!Ci.nsIDOMWindowUtils)
     return;
 
   var win;
 
   if (browserWindow)
     win = content.contentWindow;
   else 
     win = window;
   var wu = win.QueryInterface(Ci.nsIInterfaceRequestor).getInterface(Ci.nsIDOMWindowUtils);
 
-  for (var i = 0; i < renderCycles; i++) {
-    var start = Date.now();
-    for (var j = 0; j < redrawsPerSample; j++)
-      wu.redraw();
-    var end = Date.now();
+  var start = Date.now();
+  for (var j = 0; j < redrawsPerSample; j++)
+    wu.redraw();
+  var end = Date.now();
 
-    renderReport.recordTime(pageIndex, end - start);
-  }
+  renderReport.recordTime(pageIndex, end - start);
 }
 
 function plStop(force) {
   try {
     if (force == false) {
       pageIndex = 0;
-      results = {};
       if (cycle < NUM_CYCLES-1) {
-	cycle++;
-	setTimeout(plLoadPage, 0);
-	return;
+        cycle++;
+        setTimeout(plLoadPage, 0);
+        return;
       }
 
       var formats = reportFormat.split(",");
 
       for each (var fmt in formats)
-	dumpLine(report.getReport(fmt));
+        dumpLine(report.getReport(fmt));
 
       if (renderReport) {
-	dumpLine ("*************** Render report *******************");
-	for each (var fmt in formats)
-	  dumpLine(renderReport.getReport(fmt));
+        dumpLine ("*************** Render report *******************");
+        for each (var fmt in formats)
+          dumpLine(renderReport.getReport(fmt));
       }
     }
   } catch (e) {
     dumpLine(e);
   }
 
   if (content)
     content.removeEventListener('load', plLoadHandler, true);
 
   goQuitApplication();
 }
 
 /* Returns array */
-function plLoadURLsFromURI(uri) {
-  var data = "";
+function plLoadURLsFromURI(manifestUri) {
   var fstream = Cc["@mozilla.org/network/file-input-stream;1"]
     .createInstance(Ci.nsIFileInputStream);
-  var sstream = Cc["@mozilla.org/scriptableinputstream;1"]
-    .createInstance(Ci.nsIScriptableInputStream);
-  var uriFile = uri.QueryInterface(Ci.nsIFileURL);
+  var uriFile = manifestUri.QueryInterface(Ci.nsIFileURL);
+
   fstream.init(uriFile.file, -1, 0, 0);
-  sstream.init(fstream); 
-    
-  var str = sstream.read(4096);
-  while (str.length > 0) {
-    data += str;
-    str = sstream.read(4096);
-  }
-    
-  sstream.close();
-  fstream.close();
-  var p = data.split("\n");
+  var lstream = fstream.QueryInterface(Ci.nsILineInputStream);
+
+  var d = [];
+
+  var lineNo = 0;
+  var line = {value:null};
+  var more;
+  do {
+    lineNo++;
+    more = lstream.readLine(line);
+    var s = line.value;
+
+    // strip comments
+    s = s.replace(/#.*/, '');
+
+    // strip leading and trailing whitespace
+    s = s.replace(/^\s*/, '').replace(/s\*$/, '');
+
+    if (!s)
+      continue;
+
+    var flags = 0;
+    var urlspec = s;
 
-  // get rid of things that start with # (comments),
-  // or that don't have the load string, if given
-  p = p.filter(function(s) {
-		 if (s == "" || s.indexOf("#") == 0)
-		   return false;
-		 if (pageFilterRegexp && !pageFilterRegexp.test(s))
-		   return false;
-		 return true;
-	       });
+    // split on whitespace, and figure out if we have any flags
+    var items = s.split(/\s+/);
+    if (items[0] == "include") {
+      if (items.length != 2) {
+        dumpLine("tp: Error on line " + lineNo + " in " + manifestUri.spec + ": include must be followed by the manifest to include!");
+        return null;
+      }
+
+      var subManifest = gIOS.newURI(items[1], null, manifestUri);
+      if (subManifest == null) {
+        dumpLine("tp: invalid URI on line " + manifestUri.spec + ":" + lineNo + " : '" + line.value + "'");
+        return null;
+      }
 
-  // stick urlPrefix to the start if necessary
-  if (urlPrefix)
-    p = p.map(function(s) { return urlPrefix + s; });
+      var subItems = plLoadURLsFromURI(subManifest);
+      if (subItems == null)
+        return null;
+      d = d.concat(subItems);
+    } else {
+      if (items.length == 2) {
+        if (items[0].indexOf("%") != -1)
+          flags |= TEST_DOES_OWN_TIMING;
 
-  return p;
+        urlspec = items[1];
+      } else if (items.length != 1) {
+        dumpLine("tp: Error on line " + lineNo + " in " + manifestUri.spec + ": whitespace must be %-escaped!");
+        return null;
+      }
+
+      var url = gIOS.newURI(urlspec, null, manifestUri);
+
+      if (pageFilterRegexp && !pageFilterRegexp.test(url.spec))
+        continue;
+
+      d.push({   url: url,
+               flags: flags });
+    }
+  } while (more);
+
+  return d;
 }
 
 function dumpLine(str) {
   dump(str);
   dump("\n");
 }
--- a/layout/tools/pageloader/report.js
+++ b/layout/tools/pageloader/report.js
@@ -108,17 +108,20 @@ function getArrayStats(ary) {
   sorted_ary = ary.concat();
   sorted_ary.sort();
   // remove longest run
   sorted_ary.pop();
   if (sorted_ary.length%2) {
     r.median = sorted_ary[(sorted_ary.length-1)/2]; 
   }else{
     var n = Math.floor(sorted_ary.length / 2);
-    r.median = (sorted_ary[n] + sorted_ary[n + 1]) / 2;
+    if (n >= sorted_ary.length)
+      r.median = sorted_ary[n];
+    else
+      r.median = (sorted_ary[n] + sorted_ary[n + 1]) / 2;
   }
 
   // ignore max value when computing mean and stddev
   if (ary.length > 1)
     r.mean = (sum - r.max) / (ary.length - 1);
   else
     r.mean = ary[0];
 
@@ -156,62 +159,74 @@ function strPad(o, len, left) {
 	str += " ";
     }
   }
 
   str += " ";
   return str;
 }
 
+function strPadFixed0(n, len, left) {
+  return strPad(n.toFixed(0), len, left);
+}
+
 Report.prototype.getReport = function(format) {
   // avg and avg median are cumulative for all the pages
   var avgs = new Array();
   var medians = new Array();
   for (var i = 0; i < this.timeVals.length; ++i) {
      avgs[i] = getArrayStats(this.timeVals[i]).mean;
      medians[i] = getArrayStats(this.timeVals[i]).median;
   }
   var avg = getArrayStats(avgs).mean;
   var avgmed = getArrayStats(medians).mean;
 
   var report;
 
-  var prefixLen = findCommonPrefixLength(pages);
+  var prefixLen = findCommonPrefixLength(this.pages);
 
   if (format == "js") {
     // output "simple" js format;
     // array of { page: "str", value: 123.4, stddev: 23.3 } objects
     report = "([";
     for (var i = 0; i < this.timeVals.length; i++) {
       var stats = getArrayStats(this.timeVals[i]);
-      report += uneval({ page: pages[i].substr(prefixLen), value: stats.mean, stddev: stats.stdd});
+      report += uneval({ page: this.pages[i].substr(prefixLen), value: stats.mean, stddev: stats.stdd});
       report += ",";
     }
     report += "])";
   } else if (format == "jsfull") {
     // output "full" js format, with raw values
   } else if (format == "text") {
     // output text format suitable for dumping
     report = "============================================================\n";
     report += "    " + strPad("Page", 40, false) + strPad("mean") + strPad("stdd") + strPad("min") + strPad("max") + "raw" + "\n";
     for (var i = 0; i < this.timeVals.length; i++) {
       var stats = getArrayStats(this.timeVals[i]);
-      report += strPad(i, 4, true) + strPad(pages[i].substr(prefixLen), 40, false) + strPad(stats.mean.toFixed(0)) + strPad(stats.stdd.toFixed(0)) + strPad(stats.min.toFixed(0)) + strPad(stats.max.toFixed(0)) + this.timeVals[i] + "\n";
+      report +=
+        strPad(i, 4, true) +
+        strPad(this.pages[i].substr(prefixLen), 40, false) +
+        strPadFixed(stats.mean) +
+        strPadFixed(stats.stdd) +
+        strPadFixed(stats.min) +
+        strPadFixed(stats.max) +
+        this.timeVals[i] +
+        "\n";
     }
     report += "============================================================\n";
   } else if (format == "tinderbox") {
     report = "__start_tp_report\n";
     report += "_x_x_mozilla_page_load,"+avgmed+",NaN,NaN\n";  // max and min are just 0, ignored
     report += "_x_x_mozilla_page_load_details,avgmedian|"+avgmed+"|average|"+avg.toFixed(2)+"|minimum|NaN|maximum|NaN|stddev|NaN";
 
     for (var i = 0; i < this.timeVals.length; i++) {
       var r = getArrayStats(this.timeVals[i]);
       report += '|'+
         i + ';'+
-        pages[i].substr(prefixLen) + ';'+
+        this.pages[i].substr(prefixLen) + ';'+
         r.median + ';'+
         r.mean + ';'+
         r.min + ';'+
         r.max + ';'+
         this.timeVals[i].join(";") +
         "\n";
     }
     report += "__end_tp_report\n";
--- a/layout/tools/pageloader/tp-cmdline.js
+++ b/layout/tools/pageloader/tp-cmdline.js
@@ -83,17 +83,16 @@ PageLoaderCmdLineHandler.prototype =
       args.startIndex = cmdLine.handleFlagWithParam("tpstart", false);
       args.endIndex = cmdLine.handleFlagWithParam("tpend", false);
       args.filter = cmdLine.handleFlagWithParam("tpfilter", false);
       args.format = cmdLine.handleFlagWithParam("tpformat", false);
       args.useBrowserChrome = cmdLine.handleFlag("tpchrome", false);
       args.doRender = cmdLine.handleFlag("tprender", false);
       args.width = cmdLine.handleFlagWithParam("tpwidth", false);
       args.height = cmdLine.handleFlagWithParam("tpheight", false);
-      args.prefix = cmdLine.handleFlagWithParam("tpprefix", false);
       args.offline = cmdLine.handleFlag("tpoffline", false);
     }
     catch (e) {
       return;
     }
 
     // get our data through xpconnect
     args.wrappedJSObject = args;
@@ -111,17 +110,16 @@ PageLoaderCmdLineHandler.prototype =
   "  -tpcycles n        Loop through pages n times\n" +
   "  -tpstart n         Start at index n in the manifest\n" +
   "  -tpend n           End with index n in the manifest\n" +
   "  -tpformat f1,f2,.. Report format(s) to use\n" +
   "  -tpchrome          Test with normal browser chrome\n" +
   "  -tprender          Run render-only benchmark for each page\n" +
   "  -tpwidth width     Width of window\n" +
   "  -tpheight height   Height of window\n" +
-  "  -tpprefix prefix   Add 'prefix' to the start of each line in the manifest\n" +
   "  -tpoffline         Force offline mode\n"
 };
 
 
 var PageLoaderCmdLineFactory =
 {
   createInstance : function(outer, iid)
   {