Bug 1258916 part 5 - Make failures.list more general (include <failure-type> and allow path pattern), and merge skip.list into it. r=dbaron
authorXidorn Quan <quanxunzhen@gmail.com>
Thu, 24 Mar 2016 10:18:26 +0800
changeset 331109 6a5e55d72b41596d83ab1bb1f9f15c6cce1fac9a
parent 331108 62f3627f60235bafaa7d9425a058c8af5ecd2a39
child 331110 eeb6c9316ff31b5a4f2aaed2db678aabdf2a97f1
push id6048
push userkmoir@mozilla.com
push dateMon, 06 Jun 2016 19:02:08 +0000
treeherdermozilla-beta@46d72a56c57d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdbaron
bugs1258916
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1258916 part 5 - Make failures.list more general (include <failure-type> and allow path pattern), and merge skip.list into it. r=dbaron MozReview-Commit-ID: L91CnB1cnxZ
layout/reftests/w3c-css/failures.list
layout/reftests/w3c-css/import-tests.py
layout/reftests/w3c-css/received/reftest.list
layout/reftests/w3c-css/skip.list
--- a/layout/reftests/w3c-css/failures.list
+++ b/layout/reftests/w3c-css/failures.list
@@ -1,20 +1,31 @@
 # LIST OF REFTEST FAILURES FROM IMPORTS
 #
 # The syntax of this file is as follows:
 #    1) All lines marked with a '#' are comments.
-#    2) A failure is denoted by a path, starting
+#    2) A failure is denoted by <failure-type>* (as described in
+#       layout/tools/reftest/README.txt) and a path pattern starting
 #       with module. E.g.:
-#         css3-multicol/block-clip-001.xht
+#       fails css-values-3/attr-*.html
+#
+# If a test matches multiple path pattern, the last one wins. Because
+# of this, an item could have zero <failure-type>, which indicates it
+# is expected to pass, and override failure rule above it.
+
+# Fuzzy
+fuzzy-if(OSX,40,6) css-values-3/ch-unit-001.html
 
 # Bug 435426
-css-values-3/attr-color-invalid-cast.html
-css-values-3/attr-color-valid.html
-css-values-3/attr-length-invalid-cast.html
-css-values-3/attr-length-valid-zero-nofallback.html
-css-values-3/attr-length-valid-zero.html
-css-values-3/attr-length-valid.html
-css-values-3/attr-px-invalid-cast.html
-css-values-3/attr-px-valid.html
+fails css-values-3/attr-*.html
+      css-values-3/attr-*-invalid-fallback.html
+      css-values-3/attr-invalid-type-???.html
+
 # Bug 1256575
-css-values-3/calc-in-media-queries-001.html
-css-values-3/calc-in-media-queries-002.html
+fails css-values-3/calc-in-media-queries-???.html
+
+# because of dynamic change
+skip css-values-3/vh_not_refreshing_on_chrome.html
+skip css-values-3/vh_not_refreshing_on_chrome_iframe.html
+
+# because of support files (in iframe subdir) not being copied (bug 1256580)
+skip css-values-3/vh-support-transform-origin.html
+skip css-values-3/vh-support-transform-translate.html
--- a/layout/reftests/w3c-css/import-tests.py
+++ b/layout/reftests/w3c-css/import-tests.py
@@ -3,16 +3,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os
 from optparse import OptionParser
 from subprocess import Popen, PIPE
 import xml.dom.minidom
 import html5lib
 import filecmp
+import fnmatch
 import shutil
 import sys
 import re
 
 # FIXME:
 #   * Import more tests rather than just the very limited set currently
 #     chosen.
 #   * Read in a (checked-in) input file with a list of test assertions
@@ -53,18 +54,17 @@ gPrefixedProperties = [
 #  "$SUBDIR_NAME": "pref($PREF_NAME, $PREF_VALUE)"
 #
 # For example, when "@supports" was behind a pref, gDefaultPreferences had:
 #  "css3-conditional": "pref(layout.css.supports-rule.enabled,true)"
 gDefaultPreferences = {
 }
 
 gLog = None
-gFailList = {}
-gSkipList = {}
+gFailList = []
 gDestPath = None
 gSrcPath = None
 support_dirs_mapped = set()
 filemap = {}
 speclinkmap = {}
 propsaddedfor = []
 tests = []
 gOptions = None
@@ -272,34 +272,34 @@ def setup_paths():
     gSubtrees = newSubtrees
 
 def setup_log():
     global gLog
     # Since we're going to commit the tests, we should also commit
     # information about where they came from.
     gLog = open(os.path.join(gDestPath, "import.log"), "wb")
 
-def read_fail_and_skip_list():
-    global gFailList, gSkipList
+def read_fail_list():
+    global gFailList
     dirname = os.path.realpath(__file__).split(os.path.sep)
     dirname = os.path.sep.join(dirname[:len(dirname)-1])
-    failListFile = open(os.path.join(dirname, "failures.list"), "rb")
-    gFailList = [x for x in [x.lstrip().rstrip() for x in failListFile] if bool(x)
-                 and not x.startswith("#")]
-    failListFile.close()
-    skipListFile = open(os.path.join(dirname, "skip.list"), "rb")
-    gSkipList = [x for x in [x.lstrip().rstrip() for x in skipListFile]
-                 if bool(x) and not x.startswith("#")]
-    skipListFile.close()
+    with open(os.path.join(dirname, "failures.list"), "rb") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            items = line.split()
+            pat = re.compile(fnmatch.translate(items.pop()))
+            gFailList.append((pat, items))
 
 def main():
-    global gDestPath, gLog, gTestfiles, gTestFlags, gFailList, gSkipList
+    global gDestPath, gLog, gTestfiles, gTestFlags, gFailList
     read_options()
     setup_paths()
-    read_fail_and_skip_list()
+    read_fail_list()
     setup_log()
     write_log_header()
     remove_existing_dirs()
     populate_test_files()
 
     for t in gTestfiles:
         add_test_items(t, spec=None)
 
@@ -309,31 +309,33 @@ def main():
     for test in tests:
         defaultPreferences = gDefaultPreferences.get(test[1].split("/")[0], None)
         if defaultPreferences != lastDefaultPreferences:
             if defaultPreferences is None:
                 listfile.write("\ndefault-preferences\n\n")
             else:
                 listfile.write("\ndefault-preferences {0}\n\n".format(defaultPreferences))
             lastDefaultPreferences = defaultPreferences
-        key = 0
+        key = 1
         while not test[key] in gTestFlags.keys() and key < len(test):
             key = key + 1
+        testType = test[key - 1]
         testFlags = gTestFlags[test[key]]
         # Replace the Windows separators if any. Our internal strings
         # all use the system separator, however the failure/skip lists
         # and reftest.list always use '/' so we fix the paths here.
         test[key] = to_unix_path_sep(test[key])
         test[key + 1] = to_unix_path_sep(test[key + 1])
         testKey = test[key]
         if 'ahem' in testFlags:
             test = ["HTTP(../../..)"] + test
-        if testKey in gFailList:
-            test = ["fails"] + test
-        if testKey in gSkipList:
-            test = ["skip"] + test
+        fail = []
+        for pattern, failureType in gFailList:
+            if pattern.match(testKey):
+                fail = failureType
+        test = fail + test
         listfile.write(" ".join(test) + "\n")
     listfile.close()
 
     gLog.close()
 
 if __name__ == '__main__':
     main()
--- a/layout/reftests/w3c-css/received/reftest.list
+++ b/layout/reftests/w3c-css/received/reftest.list
@@ -78,17 +78,17 @@ fails == css-values-3/attr-length-valid.
 fails == css-values-3/attr-px-invalid-cast.html css-values-3/200-200-green.html
 == css-values-3/attr-px-invalid-fallback.html css-values-3/200-200-green.html
 fails == css-values-3/attr-px-valid.html css-values-3/200-200-green.html
 == css-values-3/calc-in-calc.html css-values-3/all-green.html
 fails == css-values-3/calc-in-media-queries-001.html css-values-3/all-green.html
 fails == css-values-3/calc-in-media-queries-002.html css-values-3/all-green.html
 == css-values-3/calc-invalid-range-clamping.html css-values-3/200-200-green.html
 == css-values-3/calc-parenthesis-stack.html css-values-3/all-green.html
-skip == css-values-3/ch-unit-001.html css-values-3/ch-unit-001-ref.html
+fuzzy-if(OSX,40,6) == css-values-3/ch-unit-001.html css-values-3/ch-unit-001-ref.html
 == css-values-3/initial-background-color.html css-values-3/all-green.html
 == css-values-3/vh-calc-support-pct.html css-values-3/all-green.html
 == css-values-3/vh-calc-support.html css-values-3/all-green.html
 == css-values-3/vh-em-inherit.html css-values-3/all-green.html
 == css-values-3/vh-inherit.html css-values-3/all-green.html
 == css-values-3/vh-interpolate-pct.html css-values-3/all-green.html
 == css-values-3/vh-interpolate-px.html css-values-3/all-green.html
 == css-values-3/vh-interpolate-vh.html css-values-3/all-green.html
deleted file mode 100644
--- a/layout/reftests/w3c-css/skip.list
+++ /dev/null
@@ -1,22 +0,0 @@
-# LIST OF REFTEST NEED TO SKIP FROM IMPORTS
-#
-# Always consider adding tests to failures.list first if it doesn't
-# pass. This list should only be used if we are not able to make the
-# test itself work, e.g. some dynamic tests.
-#
-# The syntax of this file is as follows:
-#    1) All lines marked with a '#' are comments.
-#    2) A failure is denoted by a path, starting
-#       with module. E.g.:
-#         css3-multicol/block-clip-001.xht
-
-# because of dynamic change
-css-values-3/vh_not_refreshing_on_chrome.html
-css-values-3/vh_not_refreshing_on_chrome_iframe.html
-
-# because of support files (in iframe subdir) not being copied (bug 1256580)
-css-values-3/vh-support-transform-origin.html
-css-values-3/vh-support-transform-translate.html
-
-# because the test fails on OS X 10.10 (and probably other platforms with certain font antialiasing)
-css-values-3/ch-unit-001.html