Bug 1312574 - Use cPickle instead of json to serialize the build system's tests database. r=gps
authorChris Manchester <cmanchester@mozilla.com>
Tue, 25 Oct 2016 12:31:59 -0700
changeset 319788 12987daa4d6800a5987b0eb197d344a1a41ba916
parent 319787 2161133a13503dab01ec5a4fca688e4630d65be3
child 319789 48c39daa6910556bf069cc610814f43e5b577fa1
push id20748
push userphilringnalda@gmail.com
push dateFri, 28 Oct 2016 03:39:55 +0000
treeherderfx-team@715360440695 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1312574
milestone52.0a1
Bug 1312574 - Use cPickle instead of json to serialize the build system's tests database. r=gps MozReview-Commit-ID: 9xmUtVV6SRN
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/test/test_testing.py
python/mozbuild/mozbuild/testing.py
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
+import cPickle as pickle
 import itertools
 import json
 import os
 
 import mozpack.path as mozpath
 
 from mozbuild.backend.base import BuildBackend
 
@@ -365,29 +366,28 @@ class CommonBackend(BuildBackend):
         self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
         self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
 
         for config in self._configs:
             self.backend_input_files.add(config.source)
 
         # Write out a machine-readable file describing every test.
         topobjdir = self.environment.topobjdir
-        with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
-            json.dump(self._test_manager.tests_by_path, fh)
+        with self._write_file(mozpath.join(topobjdir, 'all-tests.pkl'), mode='rb') as fh:
+            pickle.dump(dict(self._test_manager.tests_by_path), fh, protocol=2)
 
-        with self._write_file(mozpath.join(topobjdir, 'test-defaults.json')) as fh:
-            json.dump(self._test_manager.manifest_defaults, fh)
+        with self._write_file(mozpath.join(topobjdir, 'test-defaults.pkl'), mode='rb') as fh:
+            pickle.dump(self._test_manager.manifest_defaults, fh, protocol=2)
 
-        path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
-        with self._write_file(path) as fh:
-            json.dump({k: v for k, v in self._test_manager.installs_by_path.items()
-                       if k in self._test_manager.deferred_installs},
-                      fh,
-                      sort_keys=True,
-                      indent=4)
+        path = mozpath.join(self.environment.topobjdir, 'test-installs.pkl')
+        with self._write_file(path, mode='rb') as fh:
+            pickle.dump({k: v for k, v in self._test_manager.installs_by_path.items()
+                         if k in self._test_manager.deferred_installs},
+                        fh,
+                        protocol=2)
 
         # Write out a machine-readable file describing binaries.
         with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
             d = {
                 'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
                 'programs': [p.to_dict() for p in self._binaries.programs],
             }
             json.dump(d, fh, sort_keys=True, indent=4)
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
+import cPickle as pickle
 import json
 import os
 import unittest
 
 from mozpack.manifests import (
     InstallManifest,
 )
 from mozunit import main
@@ -519,21 +520,21 @@ class TestRecursiveMakeBackend(BackendTe
         lines = [l.strip() for l in open(x_master, 'rt').readlines()]
         self.assertEqual(lines, [
             '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
             '',
             '[include:dir1/xpcshell.ini]',
             '[include:xpcshell.ini]',
         ])
 
-        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
         self.assertTrue(os.path.exists(all_tests_path))
 
-        with open(all_tests_path, 'rt') as fh:
-            o = json.load(fh)
+        with open(all_tests_path, 'rb') as fh:
+            o = pickle.load(fh)
 
             self.assertIn('xpcshell.js', o)
             self.assertIn('dir1/test_bar.js', o)
 
             self.assertEqual(len(o['xpcshell.js']), 1)
 
     def test_test_manifest_pattern_matches_recorded(self):
         """Pattern matches in test manifests' support-files should be recorded."""
@@ -545,39 +546,39 @@ class TestRecursiveMakeBackend(BackendTe
         # done.
         entries = [e for e in m._dests.keys() if '**' in e]
         self.assertEqual(len(entries), 1)
         self.assertIn('support/**', entries[0])
 
     def test_test_manifest_deffered_installs_written(self):
         """Shared support files are written to their own data file by the backend."""
         env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
-        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
         self.assertTrue(os.path.exists(all_tests_path))
-        test_installs_path = mozpath.join(env.topobjdir, 'test-installs.json')
+        test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl')
 
         with open(test_installs_path, 'r') as fh:
-            test_installs = json.load(fh)
+            test_installs = pickle.load(fh)
 
         self.assertEqual(set(test_installs.keys()),
                          set(['child/test_sub.js',
                               'child/data/**',
                               'child/another-file.sjs']))
         for key in test_installs.keys():
             self.assertIn(key, test_installs)
 
         test_files_manifest = mozpath.join(env.topobjdir,
                                            '_build_manifests',
                                            'install',
                                            '_test_files')
 
         # First, read the generated for ini manifest contents.
         m = InstallManifest(path=test_files_manifest)
 
-        # Then, synthesize one from the test-installs.json file. This should
+        # Then, synthesize one from the test-installs.pkl file. This should
         # allow us to re-create a subset of the above.
         synthesized_manifest = InstallManifest()
         for item, installs in test_installs.items():
             for install_info in installs:
                 if len(install_info) == 3:
                     synthesized_manifest.add_pattern_symlink(*install_info)
                 if len(install_info) == 2:
                     synthesized_manifest.add_symlink(*install_info)
@@ -845,21 +846,21 @@ class TestRecursiveMakeBackend(BackendTe
             # Destination and install manifest are relative to topobjdir.
             stem = '%s/android_eclipse/%s' % (env.topobjdir, project_name)
             self.assertIn(command_template % (stem, stem), lines)
 
     def test_install_manifests_package_tests(self):
         """Ensure test suites honor package_tests=False."""
         env = self._consume('test-manifests-package-tests', RecursiveMakeBackend)
 
-        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
         self.assertTrue(os.path.exists(all_tests_path))
 
-        with open(all_tests_path, 'rt') as fh:
-            o = json.load(fh)
+        with open(all_tests_path, 'rb') as fh:
+            o = pickle.load(fh)
             self.assertIn('mochitest.js', o)
             self.assertIn('not_packaged.java', o)
 
         man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
         self.assertTrue(os.path.isdir(man_dir))
 
         full = mozpath.join(man_dir, '_test_files')
         self.assertTrue(os.path.exists(full))
--- a/python/mozbuild/mozbuild/test/test_testing.py
+++ b/python/mozbuild/mozbuild/test/test_testing.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
+import cPickle as pickle
 import os
 import shutil
 import tempfile
 import unittest
 
 import mozpack.path as mozpath
 
 from mozfile import NamedTemporaryFile
@@ -16,18 +17,17 @@ from mozunit import main
 
 from mozbuild.base import MozbuildObject
 from mozbuild.testing import (
     TestMetadata,
     TestResolver,
 )
 
 
-ALL_TESTS_JSON = b'''
-{
+ALL_TESTS = {
     "accessible/tests/mochitest/actions/test_anchors.html": [
         {
             "dir_relpath": "accessible/tests/mochitest/actions",
             "expected": "pass",
             "file_relpath": "accessible/tests/mochitest/actions/test_anchors.html",
             "flavor": "a11y",
             "here": "/Users/gps/src/firefox/accessible/tests/mochitest/actions",
             "manifest": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/a11y.ini",
@@ -149,41 +149,41 @@ ALL_TESTS_JSON = b'''
             "manifest": "/home/chris/m-c/devtools/client/markupview/test/browser.ini",
             "name": "browser_markupview_copy_image_data.js",
             "path": "/home/chris/m-c/obj-dbg/_tests/testing/mochitest/browser/devtools/client/markupview/test/browser_markupview_copy_image_data.js",
             "relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
             "subsuite": "devtools",
             "tags": "devtools"
         }
    ]
-}'''.strip()
+}
 
-TEST_DEFAULTS = b'''{
-    "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\\ndata/**\\nxpcshell_updater.ini"}
-}'''
+TEST_DEFAULTS = {
+    "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\ndata/**\nxpcshell_updater.ini"}
+}
 
 
 class Base(unittest.TestCase):
     def setUp(self):
         self._temp_files = []
 
     def tearDown(self):
         for f in self._temp_files:
             del f
 
         self._temp_files = []
 
     def _get_test_metadata(self):
-        all_tests = NamedTemporaryFile()
-        all_tests.write(ALL_TESTS_JSON)
+        all_tests = NamedTemporaryFile(mode='wb')
+        pickle.dump(ALL_TESTS, all_tests)
         all_tests.flush()
         self._temp_files.append(all_tests)
 
-        test_defaults = NamedTemporaryFile()
-        test_defaults.write(TEST_DEFAULTS)
+        test_defaults = NamedTemporaryFile(mode='wb')
+        pickle.dump(TEST_DEFAULTS, test_defaults)
         test_defaults.flush()
         self._temp_files.append(test_defaults)
 
         return TestMetadata(all_tests.name, test_defaults=test_defaults.name)
 
 
 class TestTestMetadata(Base):
     def test_load(self):
@@ -246,20 +246,20 @@ class TestTestResolver(Base):
 
         for d in self._temp_dirs:
             shutil.rmtree(d)
 
     def _get_resolver(self):
         topobjdir = tempfile.mkdtemp()
         self._temp_dirs.append(topobjdir)
 
-        with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh:
-            fh.write(ALL_TESTS_JSON)
-        with open(os.path.join(topobjdir, 'test-defaults.json'), 'wt') as fh:
-            fh.write(TEST_DEFAULTS)
+        with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
+            pickle.dump(ALL_TESTS, fh)
+        with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
+            pickle.dump(TEST_DEFAULTS, fh)
 
         o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir)
 
         # Monkey patch the test resolver to avoid tests failing to find make
         # due to our fake topscrdir.
         TestResolver._run_make = lambda *a, **b: None
 
         return o._spawn(TestResolver)
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -1,15 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
-import json
+import cPickle as pickle
 import os
 import sys
 
 import mozpack.path as mozpath
 
 from mozpack.copier import FileCopier
 from mozpack.manifests import InstallManifest
 
@@ -51,22 +51,22 @@ class TestMetadata(object):
     configuration.
     """
 
     def __init__(self, all_tests, test_defaults=None):
         self._tests_by_path = OrderedDefaultDict(list)
         self._tests_by_flavor = defaultdict(set)
         self._test_dirs = set()
 
-        with open(all_tests, 'rt') as fh:
-            test_data = json.load(fh)
+        with open(all_tests, 'rb') as fh:
+            test_data = pickle.load(fh)
         defaults = None
         if test_defaults:
-            with open(test_defaults, 'rt') as fh:
-                defaults = json.load(fh)
+            with open(test_defaults, 'rb') as fh:
+                defaults = pickle.load(fh)
         for path, tests in test_data.items():
             for metadata in tests:
                 if defaults:
                     manifest = metadata['manifest']
                     manifest_defaults = defaults.get(manifest)
                     if manifest_defaults:
                         metadata = manifestparser.combine_fields(manifest_defaults,
                                                                  metadata)
@@ -173,24 +173,24 @@ class TestMetadata(object):
 class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
 
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
 
         # If installing tests is going to result in re-generating the build
         # backend, we need to do this here, so that the updated contents of
-        # all-tests.json make it to the set of tests to run.
+        # all-tests.pkl make it to the set of tests to run.
         self._run_make(target='run-tests-deps', pass_thru=True,
                        print_directory=False)
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
-                                                'all-tests.json'),
+                                                'all-tests.pkl'),
                                    test_defaults=os.path.join(self.topobjdir,
-                                                              'test-defaults.json'))
+                                                              'test-defaults.pkl'))
 
         self._test_rewrites = {
             'a11y': os.path.join(self.topobjdir, '_tests', 'testing',
                 'mochitest', 'a11y'),
             'browser-chrome': os.path.join(self.topobjdir, '_tests', 'testing',
                 'mochitest', 'browser'),
             'jetpack-package': os.path.join(self.topobjdir, '_tests', 'testing',
                 'mochitest', 'jetpack-package'),
@@ -412,19 +412,19 @@ class SupportFilesConverter(object):
                     info.installs.append((full, mozpath.normpath(dest_path)))
         return info
 
 def _resolve_installs(paths, topobjdir, manifest):
     """Using the given paths as keys, find any unresolved installs noted
     by the build backend corresponding to those keys, and add them
     to the given manifest.
     """
-    filename = os.path.join(topobjdir, 'test-installs.json')
-    with open(filename, 'r') as fh:
-        resolved_installs = json.load(fh)
+    filename = os.path.join(topobjdir, 'test-installs.pkl')
+    with open(filename, 'rb') as fh:
+        resolved_installs = pickle.load(fh)
 
     for path in paths:
         path = path[2:]
         if path not in resolved_installs:
             raise Exception('A cross-directory support file path noted in a '
                 'test manifest does not appear in any other manifest.\n "%s" '
                 'must appear in another test manifest to specify an install '
                 'for "!/%s".' % (path, path))