Bug 920849 - Part 1: Write metadata for every test file; r=ted
authorGregory Szorc <gps@mozilla.com>
Tue, 22 Oct 2013 15:51:49 -0700
changeset 165544 67e2829b7706feaad8ebddd31e968a7f469e8876
parent 165543 820620c8a288fbd4ce80da9ec48d85851a6bf30e
child 165545 30c52c82e37b7ed3a95a0e1d3b0c9959568d5c20
push id3066
push userakeybl@mozilla.com
push dateMon, 09 Dec 2013 19:58:46 +0000
treeherdermozilla-beta@a31a0dce83aa [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs920849
milestone27.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 920849 - Part 1: Write metadata for every test file; r=ted
python/mozbuild/mozbuild/backend/base.py
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -8,18 +8,21 @@ from abc import (
     ABCMeta,
     abstractmethod,
 )
 
 import os
 import sys
 import time
 
+from contextlib import contextmanager
+
 from mach.mixin.logging import LoggingMixin
 
+from ..util import FileAvoidWrite
 from ..frontend.data import (
     ReaderSummary,
     SandboxDerived,
 )
 from .configenvironment import ConfigEnvironment
 
 
 class BackendConsumeSummary(object):
@@ -225,8 +228,31 @@ class BuildBackend(LoggingMixin):
 
         This is the main method used by child classes to react to build
         metadata.
         """
 
     def consume_finished(self):
         """Called when consume() has completed handling all objects."""
 
+    @contextmanager
+    def _write_file(self, path):
+        """Context manager to write a file.
+
+        This is a glorified wrapper around FileAvoidWrite with integration to
+        update the BackendConsumeSummary on this instance.
+
+        Example usage:
+
+            with self._write_file('foo.txt') as fh:
+                fh.write('hello world')
+        """
+
+        fh = FileAvoidWrite(path)
+        yield fh
+
+        existed, updated = fh.close()
+        if not existed:
+            self.summary.created_count += 1
+        elif updated:
+            self.summary.updated_count += 1
+        else:
+            self.summary.unchanged_count += 1
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -1,19 +1,27 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
+import json
+import os
+
 import mozpack.path as mozpath
 
 from .base import BuildBackend
 
-from ..frontend.data import XPIDLFile
+from ..frontend.data import (
+    TestManifest,
+    XPIDLFile,
+)
+
+from ..util import DefaultOnReadDict
 
 
 class XPIDLManager(object):
     """Helps manage XPCOM IDLs in the context of the build system."""
     def __init__(self, config):
         self.config = config
         self.topsrcdir = config.topsrcdir
         self.topobjdir = config.topobjdir
@@ -38,21 +46,54 @@ class XPIDLManager(object):
 
         if not allow_existing and entry['basename'] in self.idls:
             raise Exception('IDL already registered: %' % entry['basename'])
 
         self.idls[entry['basename']] = entry
         self.modules.setdefault(entry['module'], set()).add(entry['root'])
 
 
+class TestManager(object):
+    """Helps hold state related to tests."""
+
+    def __init__(self, config):
+        self.config = config
+        self.topsrcdir = mozpath.normpath(config.topsrcdir)
+
+        self.tests_by_path = DefaultOnReadDict({}, global_default=[])
+
+    def add(self, t, flavor=None):
+        t = dict(t)
+        t['flavor'] = flavor
+
+        path = mozpath.normpath(t['path'])
+        assert path.startswith(self.topsrcdir)
+
+        key = path[len(self.topsrcdir)+1:]
+        t['file_relpath'] = key
+        t['dir_relpath'] = mozpath.dirname(key)
+
+        self.tests_by_path[key].append(t)
+
+
 class CommonBackend(BuildBackend):
     """Holds logic common to all build backends."""
 
     def _init(self):
         self._idl_manager = XPIDLManager(self.environment)
+        self._test_manager = TestManager(self.environment)
 
     def consume_object(self, obj):
+        if isinstance(obj, TestManifest):
+            for test in obj.tests:
+                self._test_manager.add(test, flavor=obj.flavor)
+
         if isinstance(obj, XPIDLFile):
             self._idl_manager.register_idl(obj.source_path, obj.module)
 
     def consume_finished(self):
         if len(self._idl_manager.idls):
             self._handle_idl_manager(self._idl_manager)
+
+        # Write out a machine-readable file describing every test.
+        path = os.path.join(self.environment.topobjdir, 'all-tests.json')
+        with self._write_file(path) as fh:
+            json.dump(self._test_manager.tests_by_path, fh, sort_keys=True)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -958,16 +958,17 @@ class RecursiveMakeBackend(CommonBackend
     def _process_program(self, program, backend_file):
         backend_file.write('PROGRAM = %s\n' % program)
 
     def _process_webidl_basename(self, basename):
         header = 'mozilla/dom/%sBinding.h' % os.path.splitext(basename)[0]
         self._install_manifests['dist_include'].add_optional_exists(header)
 
     def _process_test_manifest(self, obj, backend_file):
+        # Much of the logic in this function could be moved to CommonBackend.
         self.backend_input_files.add(os.path.join(obj.topsrcdir,
             obj.manifest_relpath))
 
         # Duplicate manifests may define the same file. That's OK.
         for source, dest in obj.installs.items():
             try:
                 self._install_manifests['tests'].add_symlink(source, dest)
             except ValueError:
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -315,16 +315,19 @@ class TestManifest(SandboxDerived):
         'path',
 
         # The directory where this manifest is defined.
         'directory',
 
         # The parsed manifestparser.TestManifest instance.
         'manifest',
 
+        # List of tests. Each element is a dict of metadata.
+        'tests',
+
         # The relative path of the parsed manifest within the srcdir.
         'manifest_relpath',
 
         # If this manifest is a duplicate of another one, this is the
         # manifestparser.TestManifest of the other one.
         'dupe_manifest',
     )
 
@@ -335,16 +338,17 @@ class TestManifest(SandboxDerived):
         self.path = path
         self.directory = os.path.dirname(path)
         self.manifest = manifest
         self.flavor = flavor
         self.install_prefix = install_prefix
         self.manifest_relpath = relpath
         self.dupe_manifest = dupe_manifest
         self.installs = {}
+        self.tests = []
         self.external_installs = set()
 
 
 class LocalInclude(SandboxDerived):
     """Describes an individual local include path."""
 
     __slots__ = (
         'path',
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -274,16 +274,18 @@ class TreeMetadataEmitter(LoggingMixin):
             if filter_inactive:
                 filtered = m.active_tests(disabled=False, **self.mozinfo)
 
             out_dir = mozpath.join(install_prefix, manifest_reldir)
 
             finder = FileFinder(base=manifest_dir, find_executables=False)
 
             for test in filtered:
+                obj.tests.append(test)
+
                 obj.installs[mozpath.normpath(test['path'])] = \
                     mozpath.join(out_dir, test['relpath'])
 
                 # xpcshell defines extra files to install in the
                 # "head" and "tail" lists.
                 # All manifests support support-files.
                 for thing in ('head', 'tail', 'support-files'):
                     for pattern in test.get(thing, '').split():
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
+import json
 import os
 import unittest
 
 from mozpack.manifests import (
     InstallManifest,
 )
 from mozunit import main
 
@@ -389,16 +390,27 @@ class TestRecursiveMakeBackend(BackendTe
         lines = [l.strip() for l in open(x_master, 'rt').readlines()]
         self.assertEqual(lines, [
             '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
             '',
             '[include:dir1/xpcshell.ini]',
             '[include:xpcshell.ini]',
         ])
 
+        all_tests_path = os.path.join(env.topobjdir, 'all-tests.json')
+        self.assertTrue(os.path.exists(all_tests_path))
+
+        with open(all_tests_path, 'rt') as fh:
+            o = json.load(fh)
+
+            self.assertIn('xpcshell.js', o)
+            self.assertIn('dir1/test_bar.js', o)
+
+            self.assertEqual(len(o['xpcshell.js']), 1)
+
     def test_xpidl_generation(self):
         """Ensure xpidl files and directories are written out."""
         env = self._consume('xpidl', RecursiveMakeBackend)
 
         # Install manifests should contain entries.
         install_dir = os.path.join(env.topobjdir, '_build_manifests',
             'install')
         self.assertTrue(os.path.isfile(os.path.join(install_dir, 'dist_idl')))