Bug 1265584 - Use ujson where possible for faster metadata update, r=maja_zf
authorJames Graham <james@hoppipolla.co.uk>
Thu, 10 May 2018 15:39:14 +0100
changeset 419878 605324b9d70e822a9d25e1c6ff7ca2bcbff6085c
parent 419877 cc9b28394533a30a6ae33def5ce1117882653134
child 419879 b7716dedbf5a24e0e9a940a907225f7b3dd7d35c
push id34052
push userccoroiu@mozilla.com
push dateFri, 25 May 2018 17:52:14 +0000
treeherdermozilla-central@94d7f0e1c4d0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmaja_zf
bugs1265584
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1265584 - Use ujson where possible for faster metadata update, r=maja_zf Profiling shows that switching to this library means we no longer spend most of the update time parsing json (vs 80% or so before), making other optimisations worthwhile. This is never used in automation (except wptsync) so availability of the library in the internal pypi isn't a problem. MozReview-Commit-ID: U5gabb5lz8
testing/web-platform/mach_commands.py
testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -353,17 +353,18 @@ class MachCommands(MachCommandBase):
     def run_wpt(self, **params):
         return self.run_web_platform_tests(**params)
 
     @Command("web-platform-tests-update",
              category="testing",
              parser=create_parser_update)
     def update_web_platform_tests(self, **params):
         self.setup()
-        self.virtualenv_manager.install_pip_package('html5lib==0.99')
+        self.virtualenv_manager.install_pip_package('html5lib==1.0.1')
+        self.virtualenv_manager.install_pip_package('ujson')
         self.virtualenv_manager.install_pip_package('requests')
         wpt_updater = self._spawn(WebPlatformTestsUpdater)
         return wpt_updater.run_update(**params)
 
     @Command("wpt-update",
              category="testing",
              parser=create_parser_update)
     def update_wpt(self, **params):
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
@@ -15,16 +15,23 @@ import testloader
 import wptmanifest
 import wpttest
 from vcs import git
 manifest = None  # Module that will be imported relative to test_root
 manifestitem = None
 
 logger = structuredlog.StructuredLogger("web-platform-tests")
 
+try:
+    import ujson
+except ImportError:
+    pass
+else:
+    reader.json = ujson
+
 
 def load_test_manifests(serve_root, test_paths):
     do_delayed_imports(serve_root)
     manifest_loader = testloader.ManifestLoader(test_paths, False)
     return manifest_loader.load()
 
 
 def update_expected(test_paths, serve_root, log_file_names,