servo: Merge #600 - Rewrite reftest harness and add basic example reftest (from metajack:new-reftest); r=pcwalton
authorJack Moffitt <jack@metajack.im>
Thu, 18 Jul 2013 15:25:10 -0700
changeset 333522 c7c219b8a51b91f992466af4572b7df44a9b0e33
parent 333521 bf014a4d677ca3377c386fcc4ed4add42c7987d2
child 333523 12499f24aff5f4b581f995146a0a65cf0681467f
push id36890
push usergszorc@mozilla.com
push dateFri, 03 Feb 2017 19:58:07 +0000
treeherderautoland@be030db91f00 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspcwalton
servo: Merge #600 - Rewrite reftest harness and add basic example reftest (from metajack:new-reftest); r=pcwalton This does not port the existing src/test/html/ref tests to the new framework, as it appears to me that they aren't really reftests in the sense of Gecko's reftest. This new driver uses the Gecko methodology. Currently this will pop a window for each test due to not having a headless driver yet, and #570 means that servo segfaults when it shuts down so we can't check the exit status. There's plenty to improve in the future, but this should get us started. Source-Repo: https://github.com/servo/servo Source-Revision: 41f7109c63c5b4558ea9ab1fc0c5c9f4a8294357
servo/mk/check.mk
servo/src/test/harness/reftest/reftest.rs
--- a/servo/mk/check.mk
+++ b/servo/mk/check.mk
@@ -17,17 +17,17 @@ endef
 
 
 # Testing targets
 
 servo-test: $(DEPS_servo)
 	$(RUSTC) $(RFLAGS_servo) --test -o $@ $<
 
 reftest: $(S)src/test/harness/reftest/reftest.rs servo
-	$(RUSTC) $(RFLAGS_servo) -o $@ $< -L .
+	$(RUSTC) -o $@ $<
 
 contenttest: $(S)src/test/harness/contenttest/contenttest.rs servo
 	$(RUSTC) $(RFLAGS_servo) -o $@ $< -L .
 
 
 DEPS_CHECK_TESTABLE = $(filter-out $(NO_TESTS),$(DEPS_CHECK_ALL))
 DEPS_CHECK_TARGETS_ALL = $(addprefix check-,$(DEPS_CHECK_TESTABLE))
 DEPS_CHECK_TARGETS_FAST = $(addprefix check-,$(filter-out $(SLOW_TESTS),$(DEPS_CHECK_TESTABLE)))
@@ -39,21 +39,21 @@ check-test:
 .PHONY: check
 check: $(DEPS_CHECK_TARGETS_FAST) check-servo tidy
 
 .PHONY: check-all
 check-all: $(DEPS_CHECK_TARGETS_ALL) check-servo tidy
 
 .PHONY: check-servo
 check-servo: servo-test
-	./servo-test $(TESTNAME)
+	./servo-test
 
 .PHONY: check-ref
 check-ref: reftest
-	./reftest --source-dir=$(S)/src/test/html/ref --work-dir=src/test/html/ref $(TESTNAME)
+	./reftest $(S)src/test/ref/*.list
 
 .PHONY: check-content
 check-content: contenttest
-	./contenttest --source-dir=$(S)/src/test/html/content $(TESTNAME)
+	./contenttest --source-dir=$(S)src/test/html/content $(TESTNAME)
 
 .PHONY: tidy
 tidy: 
 	python $(S)src/etc/tidy.py $(S)src
--- a/servo/src/test/harness/reftest/reftest.rs
+++ b/servo/src/test/harness/reftest/reftest.rs
@@ -3,193 +3,146 @@
 //
 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
 extern mod std;
-extern mod servo;
+extern mod extra;
 
-use std::test::{TestOpts, run_tests_console, TestDesc};
-use std::getopts::{getopts, reqopt, opt_str, fail_str};
-use os::list_dir_path;
-use servo::run_pipeline_png;
-use servo::image::base::Image;
+use std::cell::Cell;
+use std::io;
+use std::os;
+use std::run;
+use extra::digest::{Digest, DigestUtil};
+use extra::sha1::Sha1;
+use extra::test::{DynTestName, DynTestFn, TestDesc, TestOpts, TestDescAndFn};
+use extra::test::run_tests_console;
 
-fn main(args: ~[~str]) {
-    let config = parse_config(args);
-    let opts = test_options(config);
-    let tests = find_tests(config);
-    install_rasterize_py(config);
-    run_tests_console(opts, tests);
-}
+fn main() {
+    let args = os::args();
+    if args.len() < 2 {
+        println("error: at least one reftest list must be given");
+        os::set_exit_status(1);
+        return;
+    }
 
-struct Config {
-    source_dir: ~str,
-    work_dir: ~str,
-    filter: Option<~str>
-}
-
-fn parse_config(args: ~[~str]) -> Config {
-    let args = args.tail();
-    let opts = ~[reqopt(~"source-dir"), reqopt(~"work-dir")];
-    let matches = match getopts(args, opts) {
-      Ok(m) => m,
-      Err(f) => fail fail_str(f)
+    let tests = parse_lists(args.tail());
+    let test_opts = TestOpts {
+        filter: None,
+        run_ignored: false,
+        logfile: None,
+        run_tests: true,
+        run_benchmarks: false,
+        save_results: None,
+        compare_results: None,
     };
 
-    Config {
-        source_dir: opt_str(matches, ~"source-dir"),
-        work_dir: opt_str(matches, ~"work-dir"),
-        filter: if matches.free.is_empty() {
-            None
-        } else {
-            Some(matches.free.head())
-        }
-    }
-}
-
-fn test_options(config: Config) -> TestOpts {
-    {
-        filter: config.filter,
-        run_ignored: false,
-        logfile: None
-    }
-}
-
-fn find_tests(config: Config) -> ~[TestDesc] {
-    let all_files = list_dir_path(&Path(config.source_dir));
-    let html_files = all_files.filter( |file| file.to_str().ends_with(".html") );
-    return html_files.map(|file| make_test(config, (*file).to_str()) );
-}
-
-fn make_test(config: Config, file: ~str) -> TestDesc {
-    let directives = load_test_directives(file);
-
-    {
-        name: file,
-        testfn: fn~() { run_test(config, file) },
-        ignore: directives.ignore,
-        should_fail: false
+    if !run_tests_console(&test_opts, tests) {
+        os::set_exit_status(1);
     }
 }
 
-struct Directives {
-    ignore: bool
+enum ReftestKind {
+    Same,
+    Different,
+}
+
+struct Reftest {
+    name: ~str,
+    kind: ReftestKind,
+    left: ~str,
+    right: ~str,
 }
 
-fn load_test_directives(file: ~str) -> Directives {
-    let data = match io::read_whole_file_str(&Path(file)) {
-      result::Ok(data) => data,
-      result::Err(e) => fail #fmt("unable to load directives for %s: %s", file, e)
-    };
+fn parse_lists(filenames: &[~str]) -> ~[TestDescAndFn] {
+    let mut tests: ~[TestDescAndFn] = ~[];
+    for filenames.iter().advance |file| {
+        let file_path = Path(*file);
+        let contents = match io::read_whole_file_str(&file_path) {
+            Ok(x) => x,
+            Err(s) => fail!(s)
+        };
+
+        for contents.line_iter().advance |line| {
+            let parts: ~[&str] = line.split_iter(' ').filter(|p| !p.is_empty()).collect();
+
+            if parts.len() != 3 {
+                fail!(fmt!("reftest line: '%s' doesn't match 'KIND LEFT RIGHT'", line));
+            }
 
-    let mut ignore = false;
+            let kind = match parts[0] {
+                "==" => Same,
+                "!=" => Different,
+                _ => fail!(fmt!("reftest line: '%s' has invalid kind '%s'",
+                                line, parts[0]))
+            };
+            let src_dir = file_path.dirname();
+            let file_left = src_dir + "/" + parts[1];
+            let file_right = src_dir + "/" + parts[2];
+            
+            let reftest = Reftest {
+                name: parts[1] + " / " + parts[2],
+                kind: kind,
+                left: file_left,
+                right: file_right,
+            };
 
-    for str::lines(data).each |line| {
-        if is_comment(line) {
-            if line.contains("ignore") {
-                ignore = true;
-                break;
-            }
+            tests.push(make_test(reftest));
         }
     }
+    tests
+}
 
-    fn is_comment(line: ~str) -> bool {
-        line.starts_with("<!--")
-    }
-
-    return Directives {
-        ignore: ignore
+fn make_test(reftest: Reftest) -> TestDescAndFn {
+    let name = reftest.name.clone();
+    let reftest = Cell::new(reftest);
+    TestDescAndFn {
+        desc: TestDesc {
+            name: DynTestName(name),
+            ignore: false,
+            should_fail: false,
+        },
+        testfn: DynTestFn(|| {
+            check_reftest(reftest.take());
+        }),
     }
 }
 
-fn run_test(config: Config, file: ~str) {
-    let servo_image = render_servo(config, file);
-    let ref_image = render_ref(config, file);
-
-    assert servo_image.width == ref_image.width;
-    assert servo_image.height == ref_image.height;
-    #debug("image depth: ref: %?, servo: %?", ref_image.depth, servo_image.depth);
+fn check_reftest(reftest: Reftest) {
+    let options = run::ProcessOptions::new();
+    let args = ~[~"-o", ~"/tmp/reftest-left.png", reftest.left.clone()];
+    let mut process = run::Process::new("./servo", args, options);
+    let _retval = process.finish();
+    // assert!(retval == 0);
 
-    for uint::range(0, servo_image.height) |h| {
-        for uint::range(0, servo_image.width) |w| {
-            let i = (h * servo_image.width + w) * 4;
-            let servo_pixel = (
-                servo_image.data[i + 0],
-                servo_image.data[i + 1],
-                servo_image.data[i + 2],
-                servo_image.data[i + 3]
-            );
-            let ref_pixel = (
-                ref_image.data[i + 0],
-                ref_image.data[i + 1],
-                ref_image.data[i + 2],
-                ref_image.data[i + 3]
-            );
-            #debug("i: %?, x: %?, y: %?, ref: %?, servo: %?", i, w, h, ref_pixel, servo_pixel);
+    let args = ~[~"-o", ~"/tmp/reftest-right.png", reftest.right.clone()];
+    let mut process = run::Process::new("./servo", args, options);
+    let _retval = process.finish();
+    // assert!(retval == 0);
 
-            let (sr, sg, sb, sa) = servo_pixel;
-            let (rr, rg, rb, ra) = ref_pixel;
-
-            if sr != rr
-                || sg != rg
-                || sb != rb
-                || sa != ra {
-                fail #fmt("mismatched pixel. x: %?, y: %?, ref: %?, servo: %?", w, h, ref_pixel, servo_pixel)
-            }
-        }
+    // check the pngs are bit equal
+    let left_sha = calc_hash(&Path("/tmp/reftest-left.png"));
+    let right_sha = calc_hash(&Path("/tmp/reftest-right.png"));
+    assert!(left_sha.is_some());
+    assert!(right_sha.is_some());
+    match reftest.kind {
+        Same => assert!(left_sha == right_sha),
+        Different => assert!(left_sha != right_sha),
     }
 }
 
-const WIDTH: uint = 800;
-const HEIGHT: uint = 600;
-
-fn render_servo(config: Config, file: ~str) -> Image {
-    let infile = ~"file://" + os::make_absolute(&Path(file)).to_str();
-    let outfilename = Path(file).filename().get().to_str() + ".png";
-    let outfile = Path(config.work_dir).push(outfilename).to_str();
-    run_pipeline_png(infile, outfile);
-    return sanitize_image(outfile);
-}
-
-fn render_ref(config: Config, file: ~str) -> Image {
-    let infile = file;
-    let outfilename = Path(file).filename().get().to_str() + "ref..png";
-    let outfile = Path(config.work_dir).push(outfilename);
-    // After we've generated the reference image once, we don't need
-    // to keep launching Firefox
-    if !os::path_exists(&outfile) {
-        let rasterize_path = rasterize_path(config);
-        let prog = run::start_program("python", ~[rasterize_path, infile, outfile.to_str()]);
-        prog.finish();
+fn calc_hash(path: &Path) -> Option<~str> {
+    match io::file_reader(path) {
+        Err(*) => None,
+        Ok(reader) => {
+            let mut sha = Sha1::new();
+            loop {
+                let bytes = reader.read_bytes(4096);
+                sha.input(bytes);
+                if bytes.len() < 4096 { break; }
+            }
+            Some(sha.result_str())
+        }
     }
-    return sanitize_image(outfile.to_str());
-}
-
-fn sanitize_image(file: ~str) -> Image {
-    let buf = io::read_whole_file(&Path(file)).get();
-    let image = servo::image::base::load_from_memory(buf).get();
-
-    // I don't know how to precisely control the rendered height of
-    // the Firefox output, so it is larger than we want. Trim it down.
-    assert image.width == WIDTH;
-    assert image.height >= HEIGHT;
-    let data = vec::slice(image.data, 0, image.width * HEIGHT * 4);
-
-    return Image(image.width, HEIGHT, image.depth, data);
-}
-
-fn install_rasterize_py(config: Config) {
-    use io::WriterUtil;
-    let path = rasterize_path(config);
-    let writer = io::file_writer(&Path(path), ~[io::Create, io::Truncate]).get();
-    writer.write_str(rasterize_py());
-}
-
-fn rasterize_path(config: Config) -> ~str {
-    Path(config.work_dir).push(~"rasterize.py").to_str()
-}
-
-// This is the script that uses phantom.js to render pages
-fn rasterize_py() -> ~str { #include_str("rasterize.py") }
+}
\ No newline at end of file