servo: Merge #13930 - Use headless rendering for performance test (from shinglyu:perf-llvmpipe); r=aneeshusa
authorShing Lyu <shing.lyu@gmail.com>
Mon, 07 Nov 2016 01:24:06 -0600
changeset 340082 977d4aab5053926f2ae5aa0b5fb5e3f5f4427ff0
parent 340081 82c014e4573ddb35efe074b6e4a9a0b1cc6bf6d4
child 340083 bbb0c815f4132fdaa2aaafdf95a55337ee959ef2
push id31307
push usergszorc@mozilla.com
push dateSat, 04 Feb 2017 00:59:06 +0000
treeherdermozilla-central@94079d43835f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaneeshusa
servo: Merge #13930 - Use headless rendering for performance test (from shinglyu:perf-llvmpipe); r=aneeshusa <!-- Please describe your changes on the following line: --> Properly set the software rendering environment variables and use `-z` to run the performance test in headless mode. Also changed some logging format to improve the readability and reduce log size. --- <!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: --> - [x] `./mach build -d` does not report any errors - [x] `./mach test-tidy` does not report any errors - [x] These changes fix #13903 (github issue number if applicable). <!-- Either: --> - [ ] There are tests for these changes OR - [x] These changes do not require tests because need manual test <!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. --> Source-Repo: https://github.com/servo/servo Source-Revision: f5a3d68513456b991313dfac19b94d630cc326be
servo/etc/ci/performance/README.md
servo/etc/ci/performance/runner.py
servo/etc/ci/performance/test_perf.sh
servo/python/servo/testing_commands.py
--- a/servo/etc/ci/performance/README.md
+++ b/servo/etc/ci/performance/README.md
@@ -63,8 +63,17 @@ If you want to test the data submission 
 
 ## For Gecko
 
 * Install Firefox Nightly in your PATH
 * Install [jpm](https://developer.mozilla.org/en-US/Add-ons/SDK/Tools/jpm#Installation)
 * Run `jpm xpi` in the `firefox/addon` folder
 * Install the generated `xpi` file to your Firefox Nightly
 
+# Troubleshooting
+
+ If you saw this error message:
+
+```
+venv/bin/activate: line 8: _OLD_VIRTUAL_PATH: unbound variable
+```
+
+That means your `virtualenv` is too old, try run `pip install -U virtualenv` to upgrade (If you installed ubuntu's `python-virtualenv` package, uninstall it first then install it through `pip`)
--- a/servo/etc/ci/performance/runner.py
+++ b/servo/etc/ci/performance/runner.py
@@ -19,35 +19,33 @@ def load_manifest(filename):
 
 
 def parse_manifest(text):
     return filter(lambda x: x != "" and not x.startswith("#"),
                   map(lambda x: x.strip(), text.splitlines()))
 
 
 def execute_test(url, command, timeout):
-    print("Running test:")
-    print(' '.join(command))
-    print("Timeout:{}".format(timeout))
     try:
         return subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout)
     except subprocess.CalledProcessError as e:
         print("Unexpected Fail:")
         print(e)
         print("You may want to re-run the test manually:\n{}"
               .format(' '.join(command)))
     except subprocess.TimeoutExpired:
-        print("Test timeout: {}".format(url))
+        print("Test FAILED due to timeout: {}".format(url))
     return ""
 
 
 def get_servo_command(url):
     ua_script_path = "{}/user-agent-js".format(os.getcwd())
     return ["../../../target/release/servo", url,
             "--userscripts", ua_script_path,
+            "--headless",
             "-x", "-o", "output.png"]
 
 
 def get_gecko_command(url):
     return ["./firefox/firefox/firefox",
             " --display=:0", "--no-remote"
             " -profile", "./firefox/servo",
             url]
@@ -241,19 +239,22 @@ def main():
         for testcase in testcases:
             command = (["timeout", "{}s".format(args.timeout)] +
                        command_factory(testcase))
             for run in range(args.runs):
                 print("Running test {}/{} on {}".format(run + 1,
                                                         args.runs,
                                                         testcase))
                 log = execute_test(testcase, command, args.timeout)
+                print("Finished")
                 result = parse_log(log, testcase)
                 # TODO: Record and analyze other performance.timing properties
                 results += result
+            print("To reproduce the above test, run the following command:")
+            print("    {0}\n".format(' '.join(command)))
 
         print(format_result_summary(results))
         save_result_json(results, args.output_file, testcases, args.runs)
 
     except KeyboardInterrupt:
         print("Test stopped by user, saving partial result")
         save_result_json(results, args.output_file, testcases, args.runs)
 
--- a/servo/etc/ci/performance/test_perf.sh
+++ b/servo/etc/ci/performance/test_perf.sh
@@ -5,20 +5,31 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 set -o errexit
 set -o nounset
 set -o pipefail
 
 TP5N_SOURCE="https://people.mozilla.org/~jmaher/taloszips/zips/tp5n.zip"
 TP5N_PATH="page_load_test/tp5n.zip"
-if [[ ! -f "${TP5N_PATH}" ]]; then
-    wget "${TP5N_SOURCE}" -O "${TP5N_PATH}"
+
+if [[ ! -f "$(dirname "${TP5N_PATH}")/tp5n/tp5n.manifest" ]]; then
+    if [[ ! -f "${TP5N_PATH}" ]]; then
+        echo "Downloading the test cases..."
+        wget "${TP5N_SOURCE}" -O "${TP5N_PATH}"
+        echo "done"
+    else
+        echo "Found existing test cases, skipping download."
+    fi
+    echo -n "Unzipping the test cases..."
+    unzip -q -o "${TP5N_PATH}" -d "$(dirname "${TP5N_PATH}")"
+    echo "done"
+else
+    echo "Found existing test cases, skipping download and unzip."
 fi
-unzip -o "${TP5N_PATH}" -d "$(dirname "${TP5N_PATH}")"
 
 virtualenv venv --python="$(which python3)"
 PS1="" source venv/bin/activate
 # `PS1` must be defined before activating virtualenv
 pip install "treeherder-client>=3.0.0"
 
 mkdir -p servo
 mkdir -p output
--- a/servo/python/servo/testing_commands.py
+++ b/servo/python/servo/testing_commands.py
@@ -157,16 +157,18 @@ class MachCommands(CommandBase):
                 if abs_path.startswith(prefix):
                     return suite
         return None
 
     @Command('test-perf',
              description='Run the page load performance test',
              category='testing')
     def test_perf(self):
+        self.set_software_rendering_env(True)
+
         self.ensure_bootstrapped()
         env = self.build_env()
         return call(["bash", "test_perf.sh"],
                     env=env,
                     cwd=path.join("etc", "ci", "performance"))
 
     @Command('test-unit',
              description='Run unit tests',
@@ -417,41 +419,17 @@ class MachCommands(CommandBase):
             if all_handled:
                 return correct_function(**kwargs)
             else:
                 # Dispatch each test to the correct suite via test()
                 Registrar.dispatch("test", context=self.context, params=requested_paths)
 
     # Helper for test_css and test_wpt:
     def wptrunner(self, run_file, **kwargs):
-        # On Linux and mac, find the OSMesa software rendering library and
-        # add it to the dynamic linker search path.
-        if sys.platform.startswith('linux'):
-            try:
-                args = [self.get_binary_path(kwargs["release"], not kwargs["release"])]
-                osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]), "out", "lib", "gallium")
-                os.environ["LD_LIBRARY_PATH"] = osmesa_path
-                os.environ["GALLIUM_DRIVER"] = "softpipe"
-            except BuildNotFound:
-                # This can occur when cross compiling (e.g. arm64), in which case
-                # we won't run the tests anyway so can safely ignore this step.
-                pass
-        if sys.platform.startswith('darwin'):
-            try:
-                args = [self.get_binary_path(kwargs["release"], not kwargs["release"])]
-                osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]),
-                                        "out", "src", "gallium", "targets", "osmesa", ".libs")
-                glapi_path = path.join(find_dep_path_newest('osmesa-src', args[0]),
-                                       "out", "src", "mapi", "shared-glapi", ".libs")
-                os.environ["DYLD_LIBRARY_PATH"] = osmesa_path + ":" + glapi_path
-                os.environ["GALLIUM_DRIVER"] = "softpipe"
-            except BuildNotFound:
-                # This can occur when cross compiling (e.g. arm64), in which case
-                # we won't run the tests anyway so can safely ignore this step.
-                pass
+        self.set_software_rendering_env(kwargs['release'])
 
         os.environ["RUST_BACKTRACE"] = "1"
         kwargs["debug"] = not kwargs["release"]
         if kwargs.pop("chaos"):
             kwargs["debugger"] = "rr"
             kwargs["debugger_args"] = "record --chaos"
             kwargs["repeat_until_unexpected"] = True
             # TODO: Delete rr traces from green test runs?
@@ -653,16 +631,43 @@ class MachCommands(CommandBase):
             ["make", "-C", dromaeo_dir, "web"])
 
         # Check that a release servo build exists
         bin_path = path.abspath(self.get_binary_path(release, dev))
 
         return check_call(
             [run_file, "|".join(tests), bin_path, base_dir])
 
+    def set_software_rendering_env(self, use_release):
+        # On Linux and mac, find the OSMesa software rendering library and
+        # add it to the dynamic linker search path.
+        if sys.platform.startswith('linux'):
+            try:
+                args = [self.get_binary_path(use_release, not use_release)]
+                osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]), "out", "lib", "gallium")
+                os.environ["LD_LIBRARY_PATH"] = osmesa_path
+                os.environ["GALLIUM_DRIVER"] = "softpipe"
+            except BuildNotFound:
+                # This can occur when cross compiling (e.g. arm64), in which case
+                # we won't run the tests anyway so can safely ignore this step.
+                pass
+        elif sys.platform.startswith('darwin'):
+            try:
+                args = [self.get_binary_path(use_release, not use_release)]
+                osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]),
+                                        "out", "src", "gallium", "targets", "osmesa", ".libs")
+                glapi_path = path.join(find_dep_path_newest('osmesa-src', args[0]),
+                                       "out", "src", "mapi", "shared-glapi", ".libs")
+                os.environ["DYLD_LIBRARY_PATH"] = osmesa_path + ":" + glapi_path
+                os.environ["GALLIUM_DRIVER"] = "softpipe"
+            except BuildNotFound:
+                # This can occur when cross compiling (e.g. arm64), in which case
+                # we won't run the tests anyway so can safely ignore this step.
+                pass
+
 
 def create_parser_create():
     import argparse
     p = argparse.ArgumentParser()
     p.add_argument("--no-editor", action="store_true",
                    help="Don't try to open the test in an editor")
     p.add_argument("-e", "--editor", action="store", help="Editor to use")
     p.add_argument("--no-run", action="store_true",