Bug 1536762 - part3 : enable wpt tests and add 'fuzzy' comparision. r=heycam
authoralwu <alwu@mozilla.com>
Thu, 09 May 2019 18:39:36 +0000
changeset 532131 9250cfd8c4e498475dd7351aed3ca391103b4087
parent 532130 1fa720d18b38e35831de185ac2f55356c88a7d21
child 532132 f009e75ac8711195313ec4abc50602a756335a1c
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersheycam
bugs1536762
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1536762 - part3 : enable wpt tests and add 'fuzzy' comparision. r=heycam Sometime there are some color rendering differences between the test and the reference, which is small enough to be ignored because human can't be aware of that difference. Differential Revision: https://phabricator.services.mozilla.com/D27179
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/align_center.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/align_center_position_50.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/basic.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/bidi/bidi_ruby.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/bidi/u0041_first.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/bidi/u06E9_no_strong_dir.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/evil/size_90.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/evil/size_99.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/color_hex.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/color_hsla.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/color_rgba.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/cue_selector_single_colon.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/outline_properties.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/outline_shorthand.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/text-decoration_overline.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/text-decoration_overline_underline_line-through.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/text-decoration_underline.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/white-space_normal_wrapped.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/white-space_nowrap_wrapped.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/background_box.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/bold_object/bold_namespace.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/class_object/class_namespace.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/cue_func_selector_single_colon.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/italic_object/italic_namespace.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/not_allowed_properties.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/type_selector_root.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/underline_object/underline_namespace.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/voice_object/voice_namespace.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/default_styles/bold_object_default_font-style.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/default_styles/italic_object_default_font-style.html.ini
testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/default_styles/underline_object_default_font-style.html.ini
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/align_center.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/align_center.html.ini
@@ -1,2 +1,2 @@
 [align_center.html]
-  expected: FAIL
+  fuzzy: align_center-ref.html:maxDifference=0-2;totalPixels=0-5000
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/align_center_position_50.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/align_center_position_50.html.ini
@@ -1,2 +1,2 @@
 [align_center_position_50.html]
-  expected: FAIL
+  fuzzy: align_center_position_50-ref.html:maxDifference=0-2;totalPixels=0-5000
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/basic.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/basic.html.ini
@@ -1,5 +1,2 @@
 [basic.html]
-  expected: FAIL
-  disabled:
-    if debug and (os == "linux"): https://bugzilla.mozilla.org/show_bug.cgi?id=1534541
-    if (os == "win"): https://bugzilla.mozilla.org/show_bug.cgi?id=1534541
+  fuzzy: basic-ref.html:maxDifference=0-2;totalPixels=0-41200
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/bidi/bidi_ruby.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/bidi/bidi_ruby.html.ini
@@ -1,2 +1,2 @@
 [bidi_ruby.html]
-  expected: FAIL
+  fuzzy: bidi_ruby-ref.html:maxDifference=0-2;totalPixels=0-43000
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/bidi/u0041_first.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/bidi/u0041_first.html.ini
@@ -1,2 +1,2 @@
 [u0041_first.html]
-  expected: FAIL
+  fuzzy: u0041_first-ref.html:maxDifference=0-2;totalPixels=0-100
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/bidi/u06E9_no_strong_dir.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/bidi/u06E9_no_strong_dir.html.ini
@@ -1,2 +1,2 @@
 [u06E9_no_strong_dir.html]
-  expected: FAIL
+  fuzzy: u06E9_no_strong_dir-ref.html:maxDifference=0-2;totalPixels=0-43100
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/evil/size_90.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/evil/size_90.html.ini
@@ -1,4 +1,2 @@
 [size_90.html]
-  expected:
-    if (os == "android") and e10s: TIMEOUT
-    FAIL
+  fuzzy: size_90-ref.html:maxDifference=0-2;totalPixels=0-100
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/evil/size_99.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/evil/size_99.html.ini
@@ -1,4 +1,2 @@
 [size_99.html]
-  expected:
-    if (os == "android") and e10s: TIMEOUT
-    FAIL
+  fuzzy: size_99-ref.html:maxDifference=0-2;totalPixels=0-100
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/color_hex.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/color_hex.html.ini
@@ -1,2 +1,2 @@
 [color_hex.html]
-  expected: FAIL
+  fuzzy: color_hex-ref.html:maxDifference=0-2;totalPixels=0-41200
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/color_hsla.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/color_hsla.html.ini
@@ -1,2 +1,2 @@
 [color_hsla.html]
-  expected: FAIL
+  fuzzy: color_hsla-ref.html:maxDifference=0-2;totalPixels=0-41200
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/color_rgba.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/color_rgba.html.ini
@@ -1,2 +1,2 @@
 [color_rgba.html]
-  expected: FAIL
+  fuzzy: color_rgba-ref.html:maxDifference=0-2;totalPixels=0-41200
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/cue_selector_single_colon.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/cue_selector_single_colon.html.ini
@@ -1,3 +1,2 @@
 [cue_selector_single_colon.html]
-  disabled: https://bugzilla.mozilla.org/show_bug.cgi?id=1536319
-  expected: FAIL
+  fuzzy: cue_selector_single_colon-ref.html:maxDifference=0-2;totalPixels=0-1100
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/outline_properties.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/outline_properties.html.ini
@@ -1,2 +1,2 @@
 [outline_properties.html]
-  expected: FAIL
+  fuzzy: outline_properties-ref.html:maxDifference=0-2;totalPixels=0-2700
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/outline_shorthand.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/outline_shorthand.html.ini
@@ -1,2 +1,2 @@
 [outline_shorthand.html]
-  expected: FAIL
+  fuzzy: outline_shorthand-ref.html:maxDifference=0-2;totalPixels=0-2700
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/text-decoration_overline.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/text-decoration_overline.html.ini
@@ -1,2 +1,2 @@
 [text-decoration_overline.html]
-  expected: FAIL
+  fuzzy: text-decoration_overline-ref.html:maxDifference=0-2;totalPixels=0-1600
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/text-decoration_overline_underline_line-through.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/text-decoration_overline_underline_line-through.html.ini
@@ -1,2 +1,2 @@
 [text-decoration_overline_underline_line-through.html]
-  expected: FAIL
+  fuzzy: text-decoration_overline_underline_line-ref.html:maxDifference=0-2;totalPixels=0-42000
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/text-decoration_underline.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/text-decoration_underline.html.ini
@@ -1,2 +1,2 @@
 [text-decoration_underline.html]
-  expected: FAIL
+  fuzzy: text-decoration_underline-ref.html:maxDifference=0-2;totalPixels=0-42000
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/white-space_normal_wrapped.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/white-space_normal_wrapped.html.ini
@@ -1,2 +1,2 @@
 [white-space_normal_wrapped.html]
-  expected: FAIL
+  fuzzy: white-space_normal_wrapped-ref.html:maxDifference=0-2;totalPixels=0-500
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/white-space_nowrap_wrapped.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue/white-space_nowrap_wrapped.html.ini
@@ -1,2 +1,2 @@
 [white-space_nowrap_wrapped.html]
-  expected: FAIL
+  fuzzy: white-space_nowrap_wrapped-ref.html:maxDifference=0-2;totalPixels=0-500
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/background_box.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/background_box.html.ini
@@ -1,2 +1,2 @@
 [background_box.html]
-  expected: FAIL
+  fuzzy: background_box-ref.html:maxDifference=0-2;totalPixels=0-4000
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/bold_object/bold_namespace.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/bold_object/bold_namespace.html.ini
@@ -1,2 +1,2 @@
 [bold_namespace.html]
-  expected: FAIL
+  fuzzy: bold_namespace-ref.html:maxDifference=0-2;totalPixels=0-300
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/class_object/class_namespace.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/class_object/class_namespace.html.ini
@@ -1,2 +1,2 @@
 [class_namespace.html]
-  expected: FAIL
+  fuzzy: class_namespace-ref.html:maxDifference=0-2;totalPixels=0-300
deleted file mode 100644
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/cue_func_selector_single_colon.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[cue_func_selector_single_colon.html]
-  expected: FAIL
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/italic_object/italic_namespace.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/italic_object/italic_namespace.html.ini
@@ -1,2 +1,2 @@
 [italic_namespace.html]
-  expected: FAIL
+  fuzzy: italic_namespace-ref.html:maxDifference=0-2;totalPixels=0-300
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/not_allowed_properties.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/not_allowed_properties.html.ini
@@ -1,2 +1,2 @@
 [not_allowed_properties.html]
-  expected: FAIL
+  fuzzy: not_allowed_properties-ref.html:maxDifference=0-2;totalPixels=0-2200
deleted file mode 100644
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/type_selector_root.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[type_selector_root.html]
-  expected: FAIL
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/underline_object/underline_namespace.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/underline_object/underline_namespace.html.ini
@@ -1,2 +1,2 @@
 [underline_namespace.html]
-  expected: FAIL
+  fuzzy: underline_namespace-ref.html:maxDifference=0-2;totalPixels=0-300
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/voice_object/voice_namespace.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/cue_function/voice_object/voice_namespace.html.ini
@@ -1,4 +1,2 @@
 [voice_namespace.html]
-  disabled:
-    if (os == "win") and (processor == "aarch64"): https://bugzilla.mozilla.org/show_bug.cgi?id=1519947
-  expected: FAIL
+  fuzzy: voice_namespace-ref.html:maxDifference=0-2;totalPixels=0-300
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/default_styles/bold_object_default_font-style.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/default_styles/bold_object_default_font-style.html.ini
@@ -1,4 +1,2 @@
 [bold_object_default_font-style.html]
-  disabled:
-    if (os == "win") and (processor == "aarch64"): https://bugzilla.mozilla.org/show_bug.cgi?id=1534755
-  expected: FAIL
+  fuzzy: bold_object_default_font-style-ref.html:maxDifference=0-2;totalPixels=0-1000
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/default_styles/italic_object_default_font-style.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/default_styles/italic_object_default_font-style.html.ini
@@ -1,3 +1,2 @@
 [italic_object_default_font-style.html]
-  disabled: https://bugzilla.mozilla.org/show_bug.cgi?id=1536319
-  expected: FAIL
+  fuzzy: italic_object_default_font-style-ref.html:maxDifference=0-2;totalPixels=0-1800
--- a/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/default_styles/underline_object_default_font-style.html.ini
+++ b/testing/web-platform/meta/webvtt/rendering/cues-with-video/processing-model/selectors/default_styles/underline_object_default_font-style.html.ini
@@ -1,4 +1,2 @@
 [underline_object_default_font-style.html]
-  disabled:
-    if verify and (os == "mac"): fails in verify mode
-  expected: FAIL
+  fuzzy: underline_object_default_font-style-ref.html:maxDifference=0-2;totalPixels=0-1800