Bug 1264697 - Change the format of all-tests.json to reduce redundant data. r=gps
authorChris Manchester <cmanchester@mozilla.com>
Fri, 15 Apr 2016 17:20:04 -0700
changeset 331388 70bfda59e76f300f124e5f0649ccb4cb19fc4e7a
parent 331387 02101aad5897ac020691a7739b7593c5ec97978a
child 331389 9bb074d01dbd451b0f69f65b00bacf4a2fba2d5c
push id6048
push userkmoir@mozilla.com
push dateMon, 06 Jun 2016 19:02:08 +0000
treeherdermozilla-beta@46d72a56c57d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1264697
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1264697 - Change the format of all-tests.json to reduce redundant data. r=gps The format provided to the build system by the manifest parser is highly redundant: every test lists all variables for that test, and many tests use a large support-files entry in DEFAULT that ends up in individual test objects. This patch stores these DEFAULTS per-manifest rather than per-test to save disk space, resulting in about a ~22mb smaller all-tests.json file. The in-memory representation of tests is not changed by this patch, as the defaults are again propagated to individual tests as all-tests.json is read by the test resolver. MozReview-Commit-ID: CEJaevfS5s7
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/test/test_testing.py
python/mozbuild/mozbuild/testing.py
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -171,28 +171,40 @@ class TestManager(object):
 
     def __init__(self, config):
         self.config = config
         self.topsrcdir = mozpath.normpath(config.topsrcdir)
 
         self.tests_by_path = defaultdict(list)
         self.installs_by_path = defaultdict(list)
         self.deferred_installs = set()
+        self.manifest_default_support_files = {}
 
-    def add(self, t, flavor, topsrcdir):
+    def add(self, t, flavor, topsrcdir, default_supp_files):
         t = dict(t)
         t['flavor'] = flavor
 
         path = mozpath.normpath(t['path'])
         assert mozpath.basedir(path, [topsrcdir])
 
         key = path[len(topsrcdir)+1:]
         t['file_relpath'] = key
         t['dir_relpath'] = mozpath.dirname(key)
 
+        # Support files are propagated from the default section to individual
+        # tests by the manifest parser, but we end up storing a lot of
+        # redundant data due to the huge number of support files.
+        # So if we have support files that are the same as the manifest default
+        # we track that separately, per-manifest instead of per-test, to save
+        # space.
+        supp_files = t.get('support-files')
+        if supp_files and supp_files == default_supp_files:
+            self.manifest_default_support_files[t['manifest']] = default_supp_files
+            del t['support-files']
+
         self.tests_by_path[key].append(t)
 
     def add_installs(self, obj, topsrcdir):
         for src, (dest, _) in obj.installs.iteritems():
             key = src[len(topsrcdir)+1:]
             self.installs_by_path[key].append((src, dest))
         for src, pat, dest in obj.pattern_installs:
             key = mozpath.join(src[len(topsrcdir)+1:], pat)
@@ -220,17 +232,18 @@ class CommonBackend(BuildBackend):
         self._configs = set()
         self._ipdl_sources = set()
 
     def consume_object(self, obj):
         self._configs.add(obj.config)
 
         if isinstance(obj, TestManifest):
             for test in obj.tests:
-                self._test_manager.add(test, obj.flavor, obj.topsrcdir)
+                self._test_manager.add(test, obj.flavor, obj.topsrcdir,
+                                       obj.default_support_files)
             self._test_manager.add_installs(obj, obj.topsrcdir)
 
         elif isinstance(obj, XPIDLFile):
             # TODO bug 1240134 tracks not processing XPIDL files during
             # artifact builds.
             self._idl_manager.register_idl(obj)
 
         elif isinstance(obj, ConfigFileSubstitution):
@@ -359,17 +372,18 @@ class CommonBackend(BuildBackend):
         self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
 
         for config in self._configs:
             self.backend_input_files.add(config.source)
 
         # Write out a machine-readable file describing every test.
         topobjdir = self.environment.topobjdir
         with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
-            json.dump(self._test_manager.tests_by_path, fh)
+            json.dump([self._test_manager.tests_by_path,
+                       self._test_manager.manifest_default_support_files], fh)
 
         path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
         with self._write_file(path) as fh:
             json.dump({k: v for k, v in self._test_manager.installs_by_path.items()
                        if k in self._test_manager.deferred_installs},
                       fh,
                       sort_keys=True,
                       indent=4)
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -581,16 +581,21 @@ class TestManifest(ContextDerived):
         'manifest_relpath',
 
         # The relative path of the parsed manifest within the objdir.
         'manifest_obj_relpath',
 
         # If this manifest is a duplicate of another one, this is the
         # manifestparser.TestManifest of the other one.
         'dupe_manifest',
+
+        # The support files appearing in the DEFAULT section of this
+        # manifest. This enables a space optimization in all-tests.json,
+        # see the comment in mozbuild/backend/common.py.
+        'default_support_files',
     )
 
     def __init__(self, context, path, manifest, flavor=None,
             install_prefix=None, relpath=None, dupe_manifest=False):
         ContextDerived.__init__(self, context)
 
         assert flavor in all_test_flavors()
 
@@ -602,16 +607,17 @@ class TestManifest(ContextDerived):
         self.manifest_relpath = relpath
         self.manifest_obj_relpath = relpath
         self.dupe_manifest = dupe_manifest
         self.installs = {}
         self.pattern_installs = []
         self.tests = []
         self.external_installs = set()
         self.deferred_installs = set()
+        self.default_support_files = None
 
 
 class LocalInclude(ContextDerived):
     """Describes an individual local include path."""
 
     __slots__ = (
         'path',
     )
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -1075,16 +1075,18 @@ class TreeMetadataEmitter(LoggingMixin):
                 raise SandboxValidationError('Empty test manifest: %s'
                     % path, context)
 
             obj = TestManifest(context, path, mpmanifest, flavor=flavor,
                 install_prefix=install_prefix,
                 relpath=mozpath.join(manifest_reldir, mozpath.basename(path)),
                 dupe_manifest='dupe-manifest' in defaults)
 
+            obj.default_support_files = defaults.get('support-files')
+
             filtered = mpmanifest.tests
 
             # Jetpack add-on tests are expected to be generated during the
             # build process so they won't exist here.
             if flavor != 'jetpack-addon':
                 missing = [t['name'] for t in filtered if not os.path.exists(t['path'])]
                 if missing:
                     raise SandboxValidationError('Test manifest (%s) lists '
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -523,17 +523,17 @@ class TestRecursiveMakeBackend(BackendTe
             '[include:dir1/xpcshell.ini]',
             '[include:xpcshell.ini]',
         ])
 
         all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
         self.assertTrue(os.path.exists(all_tests_path))
 
         with open(all_tests_path, 'rt') as fh:
-            o = json.load(fh)
+            o, _ = json.load(fh)
 
             self.assertIn('xpcshell.js', o)
             self.assertIn('dir1/test_bar.js', o)
 
             self.assertEqual(len(o['xpcshell.js']), 1)
 
     def test_test_manifest_pattern_matches_recorded(self):
         """Pattern matches in test manifests' support-files should be recorded."""
@@ -845,23 +845,21 @@ class TestRecursiveMakeBackend(BackendTe
             # Destination and install manifest are relative to topobjdir.
             stem = '%s/android_eclipse/%s' % (env.topobjdir, project_name)
             self.assertIn(command_template % (stem, stem), lines)
 
     def test_install_manifests_package_tests(self):
         """Ensure test suites honor package_tests=False."""
         env = self._consume('test-manifests-package-tests', RecursiveMakeBackend)
 
-        tests_dir = mozpath.join(env.topobjdir, '_tests')
-
         all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
         self.assertTrue(os.path.exists(all_tests_path))
 
         with open(all_tests_path, 'rt') as fh:
-            o = json.load(fh)
+            o, _ = json.load(fh)
 
             self.assertIn('mochitest.js', o)
             self.assertIn('not_packaged.java', o)
 
         man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
         self.assertTrue(os.path.isdir(man_dir))
 
         full = mozpath.join(man_dir, '_test_files')
--- a/python/mozbuild/mozbuild/test/test_testing.py
+++ b/python/mozbuild/mozbuild/test/test_testing.py
@@ -17,17 +17,17 @@ from mozunit import main
 from mozbuild.base import MozbuildObject
 from mozbuild.testing import (
     TestMetadata,
     TestResolver,
 )
 
 
 ALL_TESTS_JSON = b'''
-{
+[{
     "accessible/tests/mochitest/actions/test_anchors.html": [
         {
             "dir_relpath": "accessible/tests/mochitest/actions",
             "expected": "pass",
             "file_relpath": "accessible/tests/mochitest/actions/test_anchors.html",
             "flavor": "a11y",
             "here": "/Users/gps/src/firefox/accessible/tests/mochitest/actions",
             "manifest": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/a11y.ini",
@@ -76,34 +76,32 @@ ALL_TESTS_JSON = b'''
             "here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit",
             "manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
             "name": "test_0201_app_launch_apply_update.js",
             "path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
             "reason": "bug 820380",
             "relpath": "test_0201_app_launch_apply_update.js",
             "run-sequentially": "Launches application.",
             "skip-if": "toolkit == 'gonk' || os == 'android'",
-            "support-files": "\\ndata/**\\nxpcshell_updater.ini",
             "tail": ""
         },
         {
             "dir_relpath": "toolkit/mozapps/update/test/unit",
             "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
             "flavor": "xpcshell",
             "generated-files": "head_update.js",
             "head": "head_update.js head2.js",
             "here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit",
             "manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
             "name": "test_0201_app_launch_apply_update.js",
             "path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
             "reason": "bug 820380",
             "relpath": "test_0201_app_launch_apply_update.js",
             "run-sequentially": "Launches application.",
             "skip-if": "toolkit == 'gonk' || os == 'android'",
-            "support-files": "\\ndata/**\\nxpcshell_updater.ini",
             "tail": ""
         }
     ],
     "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java": [
         {
             "dir_relpath": "mobile/android/tests/background/junit3/src/common",
             "file_relpath": "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java",
             "flavor": "instrumentation",
@@ -151,17 +149,19 @@ ALL_TESTS_JSON = b'''
             "manifest": "/home/chris/m-c/devtools/client/markupview/test/browser.ini",
             "name": "browser_markupview_copy_image_data.js",
             "path": "/home/chris/m-c/obj-dbg/_tests/testing/mochitest/browser/devtools/client/markupview/test/browser_markupview_copy_image_data.js",
             "relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
             "subsuite": "devtools",
             "tags": "devtools"
         }
    ]
-}'''.strip()
+}, {
+   "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": "\\ndata/**\\nxpcshell_updater.ini"
+}]'''.strip()
 
 
 class Base(unittest.TestCase):
     def setUp(self):
         self._temp_files = []
 
     def tearDown(self):
         for f in self._temp_files:
@@ -205,16 +205,26 @@ class TestTestMetadata(Base):
         self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell',
             under_path='services'))), 2)
 
     def test_resolve_multiple_paths(self):
         t = self._get_test_metadata()
         result = list(t.resolve_tests(paths=['services', 'toolkit']))
         self.assertEqual(len(result), 4)
 
+    def test_resolve_support_files(self):
+        expected_support_files = "\ndata/**\nxpcshell_updater.ini"
+        t = self._get_test_metadata()
+        result = list(t.resolve_tests(paths=['toolkit']))
+        self.assertEqual(len(result), 2)
+
+        for test in result:
+            self.assertEqual(test['support-files'],
+                             expected_support_files)
+
     def test_resolve_path_prefix(self):
         t = self._get_test_metadata()
         result = list(t.resolve_tests(paths=['image']))
         self.assertEqual(len(result), 1)
 
 
 class TestTestResolver(Base):
     FAKE_TOPSRCDIR = '/Users/gps/src/firefox'
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -54,20 +54,28 @@ class TestMetadata(object):
 
     def __init__(self, filename=None):
         self._tests_by_path = OrderedDefaultDict(list)
         self._tests_by_flavor = defaultdict(set)
         self._test_dirs = set()
 
         if filename:
             with open(filename, 'rt') as fh:
-                d = json.load(fh)
+                test_data, manifest_support_files = json.load(fh)
 
-                for path, tests in d.items():
+                for path, tests in test_data.items():
                     for metadata in tests:
+                        # Many tests inherit their "support-files" from a manifest
+                        # level default, so we store these separately to save
+                        # disk space, and propagate them to each test when
+                        # de-serializing here.
+                        manifest = metadata['manifest']
+                        support_files = manifest_support_files.get(manifest)
+                        if support_files and 'support-files' not in metadata:
+                            metadata['support-files'] = support_files
                         self._tests_by_path[path].append(metadata)
                         self._test_dirs.add(os.path.dirname(path))
 
                         flavor = metadata.get('flavor')
                         self._tests_by_flavor[flavor].add(path)
 
     def tests_with_flavor(self, flavor):
         """Obtain all tests having the specified flavor.