Bug 473817 - allow conditional 'include' in reftest manifests. r=dbaron, a=test-only
authorJoel Maher <jmaher@mozilla.com>
Thu, 31 Mar 2011 13:08:05 -0400
changeset 64468 c8f8da9431cbb0729fd1bdeacab38b9e4bb1a0f4
parent 64467 f34c27ea9f43c72618a33f0cf2a7d79e4945b8e6
child 64469 6f02e5c7d02c23f84e37037edeca5b09aa46ebe2
push id19352
push userjmaher@mozilla.com
push dateThu, 31 Mar 2011 17:11:23 +0000
treeherdermozilla-central@6f02e5c7d02c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdbaron, test-only
bugs473817
milestone2.2a1pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 473817 - allow conditional 'include' in reftest manifests. r=dbaron, a=test-only
layout/tools/reftest/README.txt
layout/tools/reftest/reftest.js
--- a/layout/tools/reftest/README.txt
+++ b/layout/tools/reftest/README.txt
@@ -35,17 +35,25 @@ Manifest Format
 
 The test manifest format is a plain text file.  A line starting with a
 "#" is a comment.  Lines may be commented using whitespace followed by
 a "#" and the comment.  Each non-blank line (after removal of comments)
 must be one of the following:
 
 1. Inclusion of another manifest
 
-   include <relative_path>
+   <failure-type>* include <relative_path>
+
+   <failure-type> is the same as listed below for a test item.  As for 
+   test items, multiple failure types listed on the same line are 
+   combined by using the last matching failure type listed.  However, 
+   the failure type on a manifest is combined with the failure type on 
+   the test (or on a nested manifest) with the rule that the last in the
+   following list wins:  fails, random, skip.  (In other words, skip 
+   always wins, and random beats fails.)
 
 2. A test item
 
    <failure-type>* [<http>] <type> <url> <url_ref>
 
    where
 
    a. <failure-type> (optional) is one of the following:
--- a/layout/tools/reftest/reftest.js
+++ b/layout/tools/reftest/reftest.js
@@ -116,16 +116,22 @@ var gSlowestTestURL;
 var gDrawWindowFlags;
 
 const TYPE_REFTEST_EQUAL = '==';
 const TYPE_REFTEST_NOTEQUAL = '!=';
 const TYPE_LOAD = 'load';     // test without a reference (just test that it does
                               // not assert, crash, hang, or leak)
 const TYPE_SCRIPT = 'script'; // test contains individual test results
 
+// The order of these constants matters, since when we have a status
+// listed for a *manifest*, we combine the status with the status for
+// the test by using the *larger*.  
+// FIXME: In the future, we may also want to use this rule for combining
+// statuses that are on the same line (rather than making the last one
+// win).
 const EXPECTED_PASS = 0;
 const EXPECTED_FAIL = 1;
 const EXPECTED_RANDOM = 2;
 const EXPECTED_DEATH = 3;  // test must be skipped to avoid e.g. crash/hang
 
 const gProtocolRE = /^\w+:/;
 
 var HTTP_SERVER_PORT = 4444;
@@ -500,22 +506,22 @@ function BuildConditionSandbox(aURL) {
 }
 
 function ReadTopManifest(aFileURL)
 {
     gURLs = new Array();
     var url = gIOService.newURI(aFileURL, null, null);
     if (!url)
       throw "Expected a file or http URL for the manifest.";
-    ReadManifest(url);
+    ReadManifest(url, EXPECTED_PASS);
 }
 
 // Note: If you materially change the reftest manifest parsing,
 // please keep the parser in print-manifest-dirs.py in sync.
-function ReadManifest(aURL)
+function ReadManifest(aURL, inherited_status)
 {
     var secMan = CC[NS_SCRIPTSECURITYMANAGER_CONTRACTID]
                      .getService(CI.nsIScriptSecurityManager);
 
     var listURL = aURL;
     var channel = gIOService.newChannelFromURI(aURL);
     var inputStream = channel.open();
     if (channel instanceof Components.interfaces.nsIHttpChannel
@@ -553,16 +559,17 @@ function ReadManifest(aURL)
         }
 
         var expected_status = EXPECTED_PASS;
         var allow_silent_fail = false;
         var minAsserts = 0;
         var maxAsserts = 0;
         var needs_focus = false;
         var slow = false;
+        
         while (items[0].match(/^(fails|needs-focus|random|skip|asserts|slow|silentfail)/)) {
             var item = items.shift();
             var stat;
             var cond;
             var m = item.match(/^(fails|random|skip|silentfail)-if(\(.*\))$/);
             if (m) {
                 stat = m[1];
                 // Note: m[2] contains the parentheses, and we want them.
@@ -608,16 +615,18 @@ function ReadManifest(aURL)
                 } else if (stat == "skip") {
                     expected_status = EXPECTED_DEATH;
                 } else if (stat == "silentfail") {
                     allow_silent_fail = true;
                 }
             }
         }
 
+        expected_status = Math.max(expected_status, inherited_status);
+
         if (minAsserts > maxAsserts) {
             throw "Bad range in manifest file " + aURL.spec + " line " + lineNo;
         }
 
         var runHttp = false;
         var httpDepth;
         if (items[0] == "HTTP") {
             runHttp = (aURL.scheme == "file"); // We can't yet run the local HTTP server
@@ -644,17 +653,17 @@ function ReadManifest(aURL)
         }
 
         if (items[0] == "include") {
             if (items.length != 2 || runHttp)
                 throw "Error 2 in manifest file " + aURL.spec + " line " + lineNo;
             var incURI = gIOService.newURI(items[1], null, listURL);
             secMan.checkLoadURI(aURL, incURI,
                                 CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
-            ReadManifest(incURI);
+            ReadManifest(incURI, expected_status);
         } else if (items[0] == TYPE_LOAD) {
             if (items.length != 2 ||
                 (expected_status != EXPECTED_PASS &&
                  expected_status != EXPECTED_DEATH))
                 throw "Error 3 in manifest file " + aURL.spec + " line " + lineNo;
             var [testURI] = runHttp
                             ? ServeFiles(aURL, httpDepth,
                                          listURL, [items[1]])