Bug 939080 - Allow support-files in manifests to exist in parent paths; r=ted
authorGregory Szorc <gps@mozilla.com>
Mon, 18 Nov 2013 11:55:33 -0800
changeset 169389 15c695281ac7b6e648efe94dfe3a7975be586e6b
parent 169388 db98e96431baeac1ce6e5374f5a5fc0c54674cce
child 169390 24b72a63ce26db9d06d97bbe5db8d1f0a9934bbe
push id270
push userpvanderbeken@mozilla.com
push dateThu, 06 Mar 2014 09:24:21 +0000
reviewersted
bugs939080
milestone30.0a1
Bug 939080 - Allow support-files in manifests to exist in parent paths; r=ted
build/docs/test_manifests.rst
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest1.ini
python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest2.ini
python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/moz.build
python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/test_foo.js
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/mochitest.ini
python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/test_foo.js
python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/moz.build
python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/support-file.txt
python/mozbuild/mozbuild/test/frontend/test_emitter.py
toolkit/content/tests/chrome/Makefile.in
--- a/build/docs/test_manifests.rst
+++ b/build/docs/test_manifests.rst
@@ -171,17 +171,23 @@ expression syntax until it is documented
 
 File Installation
 -----------------
 
 Files referenced by manifests are automatically installed into the object
 directory into paths defined in
 :py:func:`mozbuild.frontend.emitter.TreeMetadataEmitter._process_test_manifest`.
 
-Referenced files in the manifest not in the same directory tree as the manifest
-file are **not** installed.
+Relative paths resolving to parent directory (e.g.
+``support-files = ../foo.txt`` have special behavior.
+
+For ``support-files``, the file will be installed to the default destination
+for that manifest. Only the file's base name is used to construct the final
+path: directories are irrelevant.
+
+For all other entry types, the file installation is skipped.
 
 .. _reftest_manifests:
 
 Reftest Manifests
 =================
 
 See `MDN <https://developer.mozilla.org/en-US/docs/Creating_reftest-based_unit_tests>`_.
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1037,22 +1037,24 @@ class RecursiveMakeBackend(CommonBackend
     def _process_host_simple_program(self, program, backend_file):
         backend_file.write('HOST_SIMPLE_PROGRAMS += %s\n' % program)
 
     def _process_test_manifest(self, obj, backend_file):
         # Much of the logic in this function could be moved to CommonBackend.
         self.backend_input_files.add(mozpath.join(obj.topsrcdir,
             obj.manifest_relpath))
 
-        # Duplicate manifests may define the same file. That's OK.
-        for source, dest in obj.installs.items():
+        # Don't allow files to be defined multiple times unless it is allowed.
+        # We currently allow duplicates for non-test files or test files if
+        # the manifest is listed as a duplicate.
+        for source, (dest, is_test) in obj.installs.items():
             try:
                 self._install_manifests['tests'].add_symlink(source, dest)
             except ValueError:
-                if not obj.dupe_manifest:
+                if not obj.dupe_manifest and is_test:
                     raise
 
         for base, pattern, dest in obj.pattern_installs:
             try:
                 self._install_manifests['tests'].add_pattern_symlink(base,
                     pattern, dest)
             except ValueError:
                 if not obj.dupe_manifest:
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -386,17 +386,19 @@ class LibraryDefinition(SandboxDerived):
 class TestManifest(SandboxDerived):
     """Represents a manifest file containing information about tests."""
 
     __slots__ = (
         # The type of test manifest this is.
         'flavor',
 
         # Maps source filename to destination filename. The destination
-        # path is relative from the tests root directory.
+        # path is relative from the tests root directory. Values are 2-tuples
+        # of (destpath, is_test_file) where the 2nd item is True if this
+        # item represents a test file (versus a support file).
         'installs',
 
         # A list of pattern matching installs to perform. Entries are
         # (base, pattern, dest).
         'pattern_installs',
 
         # Where all files for this manifest flavor are installed in the unified
         # test package directory.
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -485,38 +485,57 @@ class TreeMetadataEmitter(LoggingMixin):
                         # We only support globbing on support-files because
                         # the harness doesn't support * for head and tail.
                         if '*' in pattern and thing == 'support-files':
                             obj.pattern_installs.append(
                                 (manifest_dir, pattern, out_dir))
                         else:
                             full = mozpath.normpath(mozpath.join(manifest_dir,
                                 pattern))
-                            # Only install paths in our directory. This
-                            # rule is somewhat arbitrary and could be lifted.
+
+                            dest_path = mozpath.join(out_dir, pattern)
+
+                            # If the path resolves to a different directory
+                            # tree, we take special behavior depending on the
+                            # entry type.
                             if not full.startswith(manifest_dir):
-                                continue
+                                # If it's a support file, we install the file
+                                # into the current destination directory.
+                                # This implementation makes installing things
+                                # with custom prefixes impossible. If this is
+                                # needed, we can add support for that via a
+                                # special syntax later.
+                                if thing == 'support-files':
+                                    dest_path = mozpath.join(out_dir,
+                                        os.path.basename(pattern))
+                                # If it's not a support file, we ignore it.
+                                # This preserves old behavior so things like
+                                # head files doesn't get installed multiple
+                                # times.
+                                else:
+                                    continue
 
-                            obj.installs[full] = mozpath.join(out_dir, pattern)
+                            obj.installs[full] = (mozpath.normpath(dest_path),
+                                False)
 
             for test in filtered:
                 obj.tests.append(test)
 
                 obj.installs[mozpath.normpath(test['path'])] = \
-                    mozpath.join(out_dir, test['relpath'])
+                    (mozpath.join(out_dir, test['relpath']), True)
 
                 process_support_files(test)
 
             if not filtered:
                 # If there are no tests, look for support-files under DEFAULT.
                 process_support_files(defaults)
 
             # We also copy the manifest into the output directory.
             out_path = mozpath.join(out_dir, mozpath.basename(manifest_path))
-            obj.installs[path] = out_path
+            obj.installs[path] = (out_path, False)
 
             # Some manifests reference files that are auto generated as
             # part of the build or shouldn't be installed for some
             # reason. Here, we prune those files from the install set.
             # FUTURE we should be able to detect autogenerated files from
             # other build metadata. Once we do that, we can get rid of this.
             for f in defaults.get('generated-files', '').split():
                 # We re-raise otherwise the stack trace isn't informative.
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest1.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+support-files = support-file.txt
+
+[test_foo.js]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest2.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+support-files = support-file.txt
+
+[test_bar.js]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/moz.build
@@ -0,0 +1,7 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+MOCHITEST_MANIFESTS += [
+    'mochitest1.ini',
+    'mochitest2.ini',
+]
new file mode 100644
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -611,16 +611,25 @@ class TestRecursiveMakeBackend(BackendTe
 
         with open(os.path.join(env.topobjdir, 'backend.mk'), 'rb') as fh:
             lines = fh.readlines()
 
         lines = [line.rstrip() for line in lines]
 
         self.assertIn('JAR_MANIFEST := %s/jar.mn' % env.topsrcdir, lines)
 
+    def test_test_manifests_duplicate_support_files(self):
+        """Ensure duplicate support-files in test manifests work."""
+        env = self._consume('test-manifests-duplicate-support-files',
+            RecursiveMakeBackend)
+
+        p = os.path.join(env.topobjdir, '_build_manifests', 'install', 'tests')
+        m = InstallManifest(p)
+        self.assertIn('testing/mochitest/tests/support-file.txt', m)
+
     def test_android_eclipse(self):
         env = self._consume('android_eclipse', RecursiveMakeBackend)
 
         with open(mozpath.join(env.topobjdir, 'backend.mk'), 'rb') as fh:
             lines = fh.readlines()
 
         lines = [line.rstrip() for line in lines]
 
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/mochitest.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+support-files = ../support-file.txt
+
+[test_foo.js]
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/moz.build
@@ -0,0 +1,4 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+MOCHITEST_MANIFESTS += ['child/mochitest.ini']
new file mode 100644
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -335,65 +335,65 @@ class TestEmitterBasic(unittest.TestCase
                 if isinstance(o, TestManifest)]
 
         self.assertEqual(len(objs), 6)
 
         metadata = {
             'a11y.ini': {
                 'flavor': 'a11y',
                 'installs': {
-                    'a11y.ini',
-                    'test_a11y.js',
+                    'a11y.ini': False,
+                    'test_a11y.js': True,
                 },
                 'pattern-installs': 1,
             },
             'browser.ini': {
                 'flavor': 'browser-chrome',
                 'installs': {
-                    'browser.ini',
-                    'test_browser.js',
-                    'support1',
-                    'support2',
+                    'browser.ini': False,
+                    'test_browser.js': True,
+                    'support1': False,
+                    'support2': False,
                 },
             },
             'metro.ini': {
                 'flavor': 'metro-chrome',
                 'installs': {
-                    'metro.ini',
-                    'test_metro.js',
+                    'metro.ini': False,
+                    'test_metro.js': True,
                 },
             },
             'mochitest.ini': {
                 'flavor': 'mochitest',
                 'installs': {
-                    'mochitest.ini',
-                    'test_mochitest.js',
+                    'mochitest.ini': False,
+                    'test_mochitest.js': True,
                 },
                 'external': {
                     'external1',
                     'external2',
                 },
             },
             'chrome.ini': {
                 'flavor': 'chrome',
                 'installs': {
-                    'chrome.ini',
-                    'test_chrome.js',
+                    'chrome.ini': False,
+                    'test_chrome.js': True,
                 },
             },
             'xpcshell.ini': {
                 'flavor': 'xpcshell',
                 'dupe': True,
                 'installs': {
-                    'xpcshell.ini',
-                    'test_xpcshell.js',
-                    'head1',
-                    'head2',
-                    'tail1',
-                    'tail2',
+                    'xpcshell.ini': False,
+                    'test_xpcshell.js': True,
+                    'head1': False,
+                    'head2': False,
+                    'tail1': False,
+                    'tail2': False,
                 },
             },
         }
 
         for o in objs:
             m = metadata[mozpath.basename(o.manifest_relpath)]
 
             self.assertTrue(o.path.startswith(o.directory))
@@ -402,19 +402,20 @@ class TestEmitterBasic(unittest.TestCase
 
             external_normalized = set(mozpath.basename(p) for p in
                     o.external_installs)
             self.assertEqual(external_normalized, m.get('external', set()))
 
             self.assertEqual(len(o.installs), len(m['installs']))
             for path in o.installs.keys():
                 self.assertTrue(path.startswith(o.directory))
-                path = path[len(o.directory)+1:]
+                relpath = path[len(o.directory)+1:]
 
-                self.assertIn(path, m['installs'])
+                self.assertIn(relpath, m['installs'])
+                self.assertEqual(o.installs[path][1], m['installs'][relpath])
 
             if 'pattern-installs' in m:
                 self.assertEqual(len(o.pattern_installs), m['pattern-installs'])
 
     def test_test_manifest_unmatched_generated(self):
         reader = self.reader('test-manifest-unmatched-generated')
 
         with self.assertRaisesRegexp(SandboxValidationError,
@@ -433,16 +434,32 @@ class TestEmitterBasic(unittest.TestCase
         self.assertEqual(len(objs), 1)
 
         o = objs[0]
 
         self.assertEqual(o.flavor, 'mochitest')
         basenames = set(mozpath.basename(k) for k in o.installs.keys())
         self.assertEqual(basenames, {'mochitest.ini', 'test_active.html'})
 
+    def test_test_manifest_parent_support_files_dir(self):
+        """support-files referencing a file in a parent directory works."""
+        reader = self.reader('test-manifest-parent-support-files-dir')
+
+        objs = [o for o in self.read_topsrcdir(reader)
+                if isinstance(o, TestManifest)]
+
+        self.assertEqual(len(objs), 1)
+
+        o = objs[0]
+
+        expected = mozpath.join(o.srcdir, 'support-file.txt')
+        self.assertIn(expected, o.installs)
+        self.assertEqual(o.installs[expected],
+            ('testing/mochitest/tests/child/support-file.txt', False))
+
     def test_ipdl_sources(self):
         reader = self.reader('ipdl_sources')
         objs = self.read_topsrcdir(reader)
 
         ipdls = []
         for o in objs:
             if isinstance(o, IPDLFile):
                 ipdls.append('%s/%s' % (o.relativedir, o.basename))
--- a/toolkit/content/tests/chrome/Makefile.in
+++ b/toolkit/content/tests/chrome/Makefile.in
@@ -1,17 +1,12 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-MOCHITEST_CHROME_FILES := \
-  ../widgets/popup_shared.js \
-  ../widgets/tree_shared.js \
-  $(SHULL)
-
 # test_panel_focus.xul won't work if the Full Keyboard Access preference is set to
 # textboxes and lists only, so skip this test on Mac
 ifneq (cocoa,$(MOZ_WIDGET_TOOLKIT))
 MOCHITEST_CHROME_FILES += test_panel_focus.xul \
                window_panel_focus.xul \
                test_chromemargin.xul \
                window_chromemargin.xul \
                bug451540_window.xul \