bug 696585 - support pref(...) annotations in reftest manifest. r=dbaron
authorJonathan Kew <jfkthame@gmail.com>
Thu, 29 Dec 2011 00:24:48 +0000
changeset 84705 f730d670f5c4757f0762f21e1a8840f56f4ea25b
parent 84704 c05d47054f2be57cfec5c812661f09d92b9e87e1
child 84706 54bf8b9d80bedd998eb9e8c5413c3079c1204a6c
push id805
push userakeybl@mozilla.com
push dateWed, 01 Feb 2012 18:17:35 +0000
treeherdermozilla-aurora@6fb3bf232436 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdbaron
bugs696585
milestone12.0a1
bug 696585 - support pref(...) annotations in reftest manifest. r=dbaron
layout/reftests/reftest-sanity/font-default.html
layout/reftests/reftest-sanity/font-download.html
layout/reftests/reftest-sanity/font-sans-serif.html
layout/reftests/reftest-sanity/font-serif.html
layout/reftests/reftest-sanity/font-size-16.html
layout/reftests/reftest-sanity/font-size-24.html
layout/reftests/reftest-sanity/reftest.list
layout/tools/reftest/README.txt
layout/tools/reftest/reftest.js
new file mode 100644
--- /dev/null
+++ b/layout/reftests/reftest-sanity/font-default.html
@@ -0,0 +1,12 @@
+<html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<style type="text/css">
+div {
+}
+</style>
+</head>
+<body>
+<div>Hello world 123</div>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/layout/reftests/reftest-sanity/font-download.html
@@ -0,0 +1,17 @@
+<html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<style type="text/css">
+@font-face {
+  font-family: foo;
+  src: url(../fonts/DejaVuSansMono.woff);
+}
+div {
+  font-family: foo;
+}
+</style>
+</head>
+<body>
+<div>Hello world 123</div>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/layout/reftests/reftest-sanity/font-sans-serif.html
@@ -0,0 +1,13 @@
+<html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<style type="text/css">
+div {
+  font-family: sans-serif;
+}
+</style>
+</head>
+<body>
+<div>Hello world 123</div>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/layout/reftests/reftest-sanity/font-serif.html
@@ -0,0 +1,13 @@
+<html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<style type="text/css">
+div {
+  font-family: serif;
+}
+</style>
+</head>
+<body>
+<div>Hello world 123</div>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/layout/reftests/reftest-sanity/font-size-16.html
@@ -0,0 +1,13 @@
+<html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<style type="text/css">
+div {
+  font-size: 16px;
+}
+</style>
+</head>
+<body>
+<div>Hello world 123</div>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/layout/reftests/reftest-sanity/font-size-24.html
@@ -0,0 +1,13 @@
+<html lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<style type="text/css">
+div {
+  font-size: 24px;
+}
+</style>
+</head>
+<body>
+<div>Hello world 123</div>
+</body>
+</html>
--- a/layout/reftests/reftest-sanity/reftest.list
+++ b/layout/reftests/reftest-sanity/reftest.list
@@ -101,8 +101,31 @@ skip-if(!browserIsRemote) == test-displa
 
 # reftest syntax: require-or
 require-or(unrecognizedCondition,skip) script scripttest-fail.html
 require-or(true&&unrecognizedCondition,skip) script scripttest-fail.html
 require-or(unrecognizedCondition&&true,skip) script scripttest-fail.html
 require-or(unrecognizedCondition,fails) script scripttest-fail.html
 require-or(true,fails) script scripttest-pass.html
 require-or(true&&true,fails) script scripttest-pass.html
+
+# tests for pref(...) syntax in manifest, including "fails" examples with incorrect prefs
+# a boolean pref
+pref(gfx.downloadable_fonts.enabled,true) HTTP(..) != font-download.html font-default.html
+pref(gfx.downloadable_fonts.enabled,false) HTTP(..) == font-download.html font-default.html
+fails pref(gfx.downloadable_fonts.enabled,0) HTTP(..) == font-download.html font-default.html
+fails pref(gfx.downloadable_fonts.enabled,"foo") HTTP(..) == font-download.html font-default.html
+# a non-existent pref
+fails pref(not.a.real.pref.name,1) == font-download.html font-default.html
+# an integer pref
+pref(font.size.variable.x-western,16) == font-size-16.html font-default.html
+pref(font.size.variable.x-western,16) != font-size-24.html font-default.html
+pref(font.size.variable.x-western,24) == font-size-24.html font-default.html
+pref(font.size.variable.x-western,24) != font-size-16.html font-default.html
+fails pref(font.size.variable.x-western,false) == font-size-16.html font-default.html
+fails pref(font.size.variable.x-western,"foo") == font-size-16.html font-default.html
+# a string pref
+pref(font.default.x-western,"serif") == font-serif.html font-default.html
+pref(font.default.x-western,"serif") != font-sans-serif.html font-default.html
+pref(font.default.x-western,"sans-serif") == font-sans-serif.html font-default.html
+pref(font.default.x-western,"sans-serif") != font-serif.html font-default.html
+fails pref(font.default.x-western,true) == font-serif.html font-default.html
+fails pref(font.default.x-western,0) == font-serif.html font-default.html
--- a/layout/tools/reftest/README.txt
+++ b/layout/tools/reftest/README.txt
@@ -47,17 +47,17 @@ 1. Inclusion of another manifest
    combined by using the last matching failure type listed.  However, 
    the failure type on a manifest is combined with the failure type on 
    the test (or on a nested manifest) with the rule that the last in the
    following list wins:  fails, random, skip.  (In other words, skip 
    always wins, and random beats fails.)
 
 2. A test item
 
-   <failure-type>* [<http>] <type> <url> <url_ref>
+   <failure-type>* <preference>* [<http>] <type> <url> <url_ref>
 
    where
 
    a. <failure-type> (optional) is one of the following:
 
       fails  The test passes if the images of the two renderings DO NOT
              meet the conditions specified in the <type>.
 
@@ -141,17 +141,32 @@ 2. A test item
       They are evaluated in a sandbox in which a limited set of
       variables are defined.  See the BuildConditionSandbox function in
       layout/tools/reftest.js for details.
 
       Examples of using conditions:
           fails-if(winWidget) == test reference
           asserts-if(cocoaWidget,2) load crashtest
 
-   b. <http>, if present, is one of the strings (sans quotes) "HTTP" or
+   b. <pref-setting> (optional) is a string of the form
+
+          pref(<name>,<value>)
+
+      where <name> is the name of a preference setting, as seen in
+      about:config, and <value> is the value to which this preference should
+      be set. <value> may be a boolean (true/false), an integer, or a
+      quoted string *without spaces*, according to the type of the preference.
+
+      The preference will be set to the specified value prior to rendering
+      the test and reference canvases, and will be restored afterwards so
+      that following tests are not affected. Note that this feature is only
+      useful for "live" preferences that take effect immediately, without
+      requiring a browser restart.
+
+   c. <http>, if present, is one of the strings (sans quotes) "HTTP" or
       "HTTP(..)" or "HTTP(../..)" or "HTTP(../../..)", etc. , indicating that
       the test should be run over an HTTP server because it requires certain
       HTTP headers or a particular HTTP status.  (Don't use this if your test
       doesn't require this functionality, because it unnecessarily slows down
       the test.)
 
       With "HTTP", HTTP tests have the restriction that any resource an HTTP
       test accesses must be accessed using a relative URL, and the test and
@@ -200,17 +215,17 @@ 2. A test item
         var outputStream = response.bodyOutputStream;
         // ...anything else you want to do, synchronously...
       }
 
       For more details on exactly which functions and properties are available
       on request/response in handleRequest, see the nsIHttpRe(quest|sponse)
       definitions in <netwerk/test/httpserver/nsIHttpServer.idl>.
 
-   c. <type> is one of the following:
+   d. <type> is one of the following:
 
       ==     The test passes if the images of the two renderings are the
              SAME.
       !=     The test passes if the images of the two renderings are 
              DIFFERENT.
       load   The test passes unconditionally if the page loads.  url_ref
              must be omitted, and the test cannot be marked as fails or
              random.  (Used to test for crashes, hangs, assertions, and
@@ -228,20 +243,20 @@ 2. A test item
              otherwise it returns false.
 
              testDescription() returns a string describing the test
              result.
 
              url_ref must be omitted. The test may be marked as fails or
              random. (Used to test the JavaScript Engine.)
 
-   d. <url> is either a relative file path or an absolute URL for the
+   e. <url> is either a relative file path or an absolute URL for the
       test page
 
-   e. <url_ref> is either a relative file path or an absolute URL for
+   f. <url_ref> is either a relative file path or an absolute URL for
       the reference page
 
    The only difference between <url> and <url_ref> is that results of
    the test are reported using <url> only.
 
 3. Specification of a url prefix
 
    url-prefix <string>
--- a/layout/tools/reftest/reftest.js
+++ b/layout/tools/reftest/reftest.js
@@ -145,16 +145,23 @@ const TYPE_SCRIPT = 'script'; // test co
 // FIXME: In the future, we may also want to use this rule for combining
 // statuses that are on the same line (rather than making the last one
 // win).
 const EXPECTED_PASS = 0;
 const EXPECTED_FAIL = 1;
 const EXPECTED_RANDOM = 2;
 const EXPECTED_DEATH = 3;  // test must be skipped to avoid e.g. crash/hang
 
+// types of preference value we might want to set for a specific test
+const PREF_BOOLEAN = 0;
+const PREF_STRING  = 1;
+const PREF_INTEGER = 2;
+
+var gPrefsToRestore = [];
+
 const gProtocolRE = /^\w+:/;
 
 var HTTP_SERVER_PORT = 4444;
 const HTTP_SERVER_PORTS_TO_TRY = 50;
 
 // whether to run slow tests or not
 var gRunSlowTests = true;
 
@@ -676,18 +683,19 @@ function ReadManifest(aURL, inherited_st
         }
 
         var expected_status = EXPECTED_PASS;
         var allow_silent_fail = false;
         var minAsserts = 0;
         var maxAsserts = 0;
         var needs_focus = false;
         var slow = false;
+        var prefSettings = [];
         
-        while (items[0].match(/^(fails|needs-focus|random|skip|asserts|slow|require-or|silentfail)/)) {
+        while (items[0].match(/^(fails|needs-focus|random|skip|asserts|slow|require-or|silentfail|pref)/)) {
             var item = items.shift();
             var stat;
             var cond;
             var m = item.match(/^(fails|random|skip|silentfail)-if(\(.*\))$/);
             if (m) {
                 stat = m[1];
                 // Note: m[2] contains the parentheses, and we want them.
                 cond = Components.utils.evalInSandbox(m[2], sandbox);
@@ -739,16 +747,34 @@ function ReadManifest(aURL, inherited_st
                 }
             } else if ((m = item.match(/^slow-if\((.*?)\)$/))) {
                 cond = false;
                 if (Components.utils.evalInSandbox("(" + m[1] + ")", sandbox))
                     slow = true;
             } else if (item == "silentfail") {
                 cond = false;
                 allow_silent_fail = true;
+            } else if ((m = item.match(/^pref\((.+?),(.*)\)$/))) {
+                cond = false;
+                var prefName = m[1];
+                var prefVal = Components.utils.evalInSandbox("(" + m[2] + ")", sandbox);
+                var prefType;
+                var valType = typeof(prefVal);
+                if (valType == "boolean") {
+                    prefType = PREF_BOOLEAN;
+                } else if (valType == "string") {
+                    prefType = PREF_STRING;
+                } else if (valType == "number" && (parseInt(prefVal) == prefVal)) {
+                    prefType = PREF_INTEGER;
+                } else {
+                    throw "Error in pref value in manifest file " + aURL.spec + " line " + lineNo;
+                }
+                prefSettings.push( { name: prefName,
+                                     type: prefType,
+                                     value: prefVal } );
             } else {
                 throw "Error 1 in manifest file " + aURL.spec + " line " + lineNo;
             }
 
             if (cond) {
                 if (stat == "fails") {
                     expected_status = EXPECTED_FAIL;
                 } else if (stat == "random") {
@@ -817,16 +843,17 @@ function ReadManifest(aURL, inherited_st
             gURLs.push( { type: TYPE_LOAD,
                           expected: expected_status,
                           allowSilentFail: allow_silent_fail,
                           prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           needsFocus: needs_focus,
                           slow: slow,
+                          prefSettings: prefSettings,
                           url1: testURI,
                           url2: null } );
         } else if (items[0] == TYPE_SCRIPT) {
             if (items.length != 2)
                 throw "Error 4 in manifest file " + aURL.spec + " line " + lineNo;
             var [testURI] = runHttp
                             ? ServeFiles(aURL, httpDepth,
                                          listURL, [items[1]])
@@ -839,16 +866,17 @@ function ReadManifest(aURL, inherited_st
             gURLs.push( { type: TYPE_SCRIPT,
                           expected: expected_status,
                           allowSilentFail: allow_silent_fail,
                           prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           needsFocus: needs_focus,
                           slow: slow,
+                          prefSettings: prefSettings,
                           url1: testURI,
                           url2: null } );
         } else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL) {
             if (items.length != 3)
                 throw "Error 5 in manifest file " + aURL.spec + " line " + lineNo;
             var [testURI, refURI] = runHttp
                                   ? ServeFiles(aURL, httpDepth,
                                                listURL, [items[1], items[2]])
@@ -864,16 +892,17 @@ function ReadManifest(aURL, inherited_st
             gURLs.push( { type: items[0],
                           expected: expected_status,
                           allowSilentFail: allow_silent_fail,
                           prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           needsFocus: needs_focus,
                           slow: slow,
+                          prefSettings: prefSettings,
                           url1: testURI,
                           url2: refURI } );
         } else {
             throw "Error 6 in manifest file " + aURL.spec + " line " + lineNo;
         }
     }
 }
 
@@ -892,17 +921,18 @@ function AddURIUseCount(uri)
 
 function BuildUseCounts()
 {
     gURIUseCounts = {};
     for (var i = 0; i < gURLs.length; ++i) {
         var url = gURLs[i];
         if (url.expected != EXPECTED_DEATH &&
             (url.type == TYPE_REFTEST_EQUAL ||
-             url.type == TYPE_REFTEST_NOTEQUAL)) {
+             url.type == TYPE_REFTEST_NOTEQUAL) &&
+            url.prefSettings.length == 0) {
             AddURIUseCount(gURLs[i].url1);
             AddURIUseCount(gURLs[i].url2);
         }
     }
 }
 
 function ServeFiles(manifestURL, depth, aURL, files)
 {
@@ -963,31 +993,104 @@ function Focus()
     }
     return true;
 }
 
 function StartCurrentTest()
 {
     gTestLog = [];
 
+    RestoreChangedPreferences();
+
     // make sure we don't run tests that are expected to kill the browser
     while (gURLs.length > 0) {
         var test = gURLs[0];
         if (test.expected == EXPECTED_DEATH) {
             ++gTestResults.Skip;
             gDumpLog("REFTEST TEST-KNOWN-FAIL | " + test.url1.spec + " | (SKIP)\n");
             gURLs.shift();
         } else if (test.needsFocus && !Focus()) {
             ++gTestResults.Skip;
             gDumpLog("REFTEST TEST-KNOWN-FAIL | " + test.url1.spec + " | (SKIPPED; COULDN'T GET FOCUS)\n");
             gURLs.shift();
         } else if (test.slow && !gRunSlowTests) {
             ++gTestResults.Slow;
             gDumpLog("REFTEST TEST-KNOWN-SLOW | " + test.url1.spec + " | (SLOW)\n");
             gURLs.shift();
+        } else if (gURLs[0].prefSettings.length > 0) {
+            var prefs = Components.classes["@mozilla.org/preferences-service;1"].
+                        getService(Components.interfaces.nsIPrefBranch2);
+            var badPref = undefined;
+            try {
+                gURLs[0].prefSettings.forEach(function(ps) {
+                    var oldVal;
+                    if (ps.type == PREF_BOOLEAN) {
+                        try {
+                            oldVal = prefs.getBoolPref(ps.name);
+                        } catch (e) {
+                            badPref = "boolean preference '" + ps.name + "'";
+                            throw "bad pref";
+                        }
+                    } else if (ps.type == PREF_STRING) {
+                        try {
+                            oldVal = prefs.getCharPref(ps.name);
+                        } catch (e) {
+                            badPref = "string preference '" + ps.name + "'";
+                            throw "bad pref";
+                        }
+                    } else if (ps.type == PREF_INTEGER) {
+                        try {
+                            oldVal = prefs.getIntPref(ps.name);
+                        } catch (e) {
+                            badPref = "integer preference '" + ps.name + "'";
+                            throw "bad pref";
+                        }
+                    } else {
+                        throw "internal error - unknown preference type";
+                    }
+                    if (oldVal != ps.value) {
+                        gPrefsToRestore.push( { name: ps.name,
+                                                type: ps.type,
+                                                value: oldVal } );
+                        var value = ps.value;
+                        if (ps.type == PREF_BOOLEAN) {
+                            prefs.setBoolPref(ps.name, value);
+                        } else if (ps.type == PREF_STRING) {
+                            prefs.setCharPref(ps.name, value);
+                            value = '"' + value + '"';
+                        } else if (ps.type == PREF_INTEGER) {
+                            prefs.setIntPref(ps.name, value);
+                        }
+                        gDumpLog("SET PREFERENCE pref(" + ps.name + "," + value + ")\n");
+                    }
+                });
+            } catch (e) {
+                if (e == "bad pref") {
+                    if (test.expected == EXPECTED_FAIL) {
+                        gDumpLog("REFTEST TEST-KNOWN-FAIL | " + test.url1.spec +
+                                 " | (SKIPPED; " + badPref + " not known or wrong type)\n");
+                        ++gTestResults.Skip;
+                    } else {
+                        gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | " + test.url1.spec +
+                                 " | " + badPref + " not known or wrong type\n");
+                        ++gTestResults.UnexpectedFail;
+                    }
+                } else {
+                    throw e;
+                }
+            }
+            if (badPref != undefined) {
+                // undo anything we changed
+                RestoreChangedPreferences();
+
+                // skip the test that had a bad preference
+                gURLs.shift();
+            } else {
+                break;
+            }
         } else {
             break;
         }
     }
 
     if (gURLs.length == 0) {
         DoneTests();
     }
@@ -999,17 +1102,18 @@ function StartCurrentTest()
     }
 }
 
 function StartCurrentURI(aState)
 {
     gState = aState;
     gCurrentURL = gURLs[0]["url" + aState].spec;
 
-    if (gURICanvases[gCurrentURL] &&
+    if (gURLs[0].prefSettings.length == 0 &&
+        gURICanvases[gCurrentURL] &&
         (gURLs[0].type == TYPE_REFTEST_EQUAL ||
          gURLs[0].type == TYPE_REFTEST_NOTEQUAL) &&
         gURLs[0].maxAsserts == 0) {
         // Pretend the document loaded --- RecordResult will notice
         // there's already a canvas for this URL
         gContainingWindow.setTimeout(RecordResult, 0);
     } else {
         var currentTest = gTotalTests - gURLs.length;
@@ -1259,17 +1363,18 @@ function RecordResult(testRunTime, error
         if (anyFailed && expected == EXPECTED_PASS) {
             FlushTestLog();
         }
 
         FinishTestItem();
         return;
     }
 
-    if (gURICanvases[gCurrentURL]) {
+    if (gURLs[0].prefSettings.length == 0 &&
+        gURICanvases[gCurrentURL]) {
         gCurrentCanvas = gURICanvases[gCurrentURL];
     }
     if (gCurrentCanvas == null) {
         gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | program error managing snapshots\n");
         ++gTestResults.Exception;
     }
     if (gState == 1) {
         gCanvas1 = gCurrentCanvas;
@@ -1338,18 +1443,20 @@ function RecordResult(testRunTime, error
                     gDumpLog("REFTEST   IMAGE: " + gCanvas1.toDataURL() + "\n");
                 }
             }
 
             if (!test_passed && expected == EXPECTED_PASS) {
                 FlushTestLog();
             }
 
-            UpdateCanvasCache(gURLs[0].url1, gCanvas1);
-            UpdateCanvasCache(gURLs[0].url2, gCanvas2);
+            if (gURLs[0].prefSettings.length == 0) {
+                UpdateCanvasCache(gURLs[0].url1, gCanvas1);
+                UpdateCanvasCache(gURLs[0].url2, gCanvas2);
+            }
 
             CleanUpCrashDumpFiles();
             FinishTestItem();
             break;
         default:
             throw "Unexpected state.";
     }
 }
@@ -1469,16 +1576,37 @@ function DoAssertionCheck(numAsserts)
 }
 
 function ResetRenderingState()
 {
     SendResetRenderingState();
     // We would want to clear any viewconfig here, if we add support for it
 }
 
+function RestoreChangedPreferences()
+{
+    if (gPrefsToRestore.length > 0) {
+        var prefs = Components.classes["@mozilla.org/preferences-service;1"].
+                    getService(Components.interfaces.nsIPrefBranch2);
+        gPrefsToRestore.forEach(function(ps) {
+            var value = ps.value;
+            if (ps.type == PREF_BOOLEAN) {
+                prefs.setBoolPref(ps.name, value);
+            } else if (ps.type == PREF_STRING) {
+                prefs.setCharPref(ps.name, value);
+                value = '"' + value + '"';
+            } else if (ps.type == PREF_INTEGER) {
+                prefs.setIntPref(ps.name, value);
+            }
+            gDumpLog("RESTORE PREFERENCE pref(" + ps.name + "," + value + ")\n");
+        });
+        gPrefsToRestore = [];
+    }
+}
+
 function RegisterMessageListenersAndLoadContentScript()
 {
     gBrowserMessageManager.addMessageListener(
         "reftest:AssertionCount",
         function (m) { RecvAssertionCount(m.json.count); }
     );
     gBrowserMessageManager.addMessageListener(
         "reftest:ContentReady",