Bug 917086 - Disallow DIRS, PARALLEL_DIRS and TEST_DIRS under TOOL_DIRS and TEST_TOOL_DIRS, and adapt moz.build files accordingly. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 19 Sep 2013 07:43:02 +0900
changeset 161628 8dd27d839acc22b5303b366afb9a3968e5e219fc
parent 161627 fdaa112654646d6f63e31216b68c29fe55e13487
child 161629 554bfe767519cb699b62a73cba8cd7d95b3bfc7f
push idunknown
push userunknown
push dateunknown
reviewersgps
bugs917086
milestone27.0a1
Bug 917086 - Disallow DIRS, PARALLEL_DIRS and TEST_DIRS under TOOL_DIRS and TEST_TOOL_DIRS, and adapt moz.build files accordingly. r=gps Also mark TOOL_DIRS/TEST_TOOL_DIRS directories in backend.mk and recurse them normally instead of forcing make -C dir libs for them.
config/recurse.mk
config/rules.mk
content/base/test/moz.build
js/src/config/recurse.mk
js/src/config/rules.mk
js/xpconnect/tests/moz.build
layout/style/test/moz.build
modules/libjar/test/moz.build
netwerk/test/moz.build
parser/htmlparser/tests/mochitest/html5lib_tree_construction/moz.build
parser/htmlparser/tests/mochitest/moz.build
parser/htmlparser/tests/moz.build
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/foo/biz/moz.build
python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/foo/moz.build
python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/moz.build
python/mozbuild/mozbuild/test/frontend/test_reader.py
rdf/tests/moz.build
toolkit/components/url-classifier/tests/moz.build
xpcom/sample/moz.build
xpcom/tests/moz.build
--- a/config/recurse.mk
+++ b/config/recurse.mk
@@ -38,15 +38,12 @@ ifdef PARALLEL_DIRS
 	+@$(MAKE) $$(PARALLEL_DIRS_$(1))
 endif
 	$$(LOOP_OVER_DIRS)
 
 endef
 
 $(foreach subtier,export compile libs tools,$(eval $(call CREATE_SUBTIER_TRAVERSAL_RULE,$(subtier))))
 
-compile export:: $(SUBMAKEFILES)
+compile export tools:: $(SUBMAKEFILES)
 	$(LOOP_OVER_TOOL_DIRS)
 
-tools:: $(SUBMAKEFILES)
-	$(foreach dir,$(TOOL_DIRS),$(call SUBMAKE,libs,$(dir)))
-
 endif
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -732,16 +732,23 @@ ifndef NO_MAKEFILE_RULE
 GLOBAL_DEPS += Makefile.in
 endif
 
 ##############################################
 compile:: $(OBJS) $(HOST_OBJS)
 
 include $(topsrcdir)/config/makefiles/target_libs.mk
 
+ifdef IS_TOOL_DIR
+# One would think "tools:: libs" would work, but it turns out that combined with
+# bug 907365, this makes make forget to run some rules sometimes.
+tools::
+	@$(MAKE) libs
+endif
+
 ##############################################
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 ifdef MOZ_PROFILE_USE
 ifeq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 # When building with PGO, we have to make sure to re-link
 # in the MOZ_PROFILE_USE phase if we linked in the
 # MOZ_PROFILE_GENERATE phase. We'll touch this pgo.relink
 # file in the link rule in the GENERATE phase to indicate
--- a/content/base/test/moz.build
+++ b/content/base/test/moz.build
@@ -1,15 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += [
+TEST_TOOL_DIRS += [
     'chrome',
     'csp',
     'websocket_hybi',
 ]
 
 MODULE = 'content'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
--- a/js/src/config/recurse.mk
+++ b/js/src/config/recurse.mk
@@ -38,15 +38,12 @@ ifdef PARALLEL_DIRS
 	+@$(MAKE) $$(PARALLEL_DIRS_$(1))
 endif
 	$$(LOOP_OVER_DIRS)
 
 endef
 
 $(foreach subtier,export compile libs tools,$(eval $(call CREATE_SUBTIER_TRAVERSAL_RULE,$(subtier))))
 
-compile export:: $(SUBMAKEFILES)
+compile export tools:: $(SUBMAKEFILES)
 	$(LOOP_OVER_TOOL_DIRS)
 
-tools:: $(SUBMAKEFILES)
-	$(foreach dir,$(TOOL_DIRS),$(call SUBMAKE,libs,$(dir)))
-
 endif
--- a/js/src/config/rules.mk
+++ b/js/src/config/rules.mk
@@ -732,16 +732,23 @@ ifndef NO_MAKEFILE_RULE
 GLOBAL_DEPS += Makefile.in
 endif
 
 ##############################################
 compile:: $(OBJS) $(HOST_OBJS)
 
 include $(topsrcdir)/config/makefiles/target_libs.mk
 
+ifdef IS_TOOL_DIR
+# One would think "tools:: libs" would work, but it turns out that combined with
+# bug 907365, this makes make forget to run some rules sometimes.
+tools::
+	@$(MAKE) libs
+endif
+
 ##############################################
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 ifdef MOZ_PROFILE_USE
 ifeq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 # When building with PGO, we have to make sure to re-link
 # in the MOZ_PROFILE_USE phase if we linked in the
 # MOZ_PROFILE_GENERATE phase. We'll touch this pgo.relink
 # file in the link rule in the GENERATE phase to indicate
--- a/js/xpconnect/tests/moz.build
+++ b/js/xpconnect/tests/moz.build
@@ -1,15 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += [
+TEST_TOOL_DIRS += [
     'idl',
     'mochitest',
     'chrome',
     'browser',
     'components/native',
     'components/js',
 ]
 
--- a/layout/style/test/moz.build
+++ b/layout/style/test/moz.build
@@ -1,13 +1,13 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['chrome']
+TEST_TOOL_DIRS += ['chrome']
 
 MODULE = 'layout'
 
 HOST_CPPSRCS += [
     'ListCSSProperties.cpp',
 ]
--- a/modules/libjar/test/moz.build
+++ b/modules/libjar/test/moz.build
@@ -1,11 +1,11 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += ['mochitest', 'chrome']
+TEST_TOOL_DIRS += ['mochitest', 'chrome']
 
 MODULE = 'test_libjar'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
--- a/netwerk/test/moz.build
+++ b/netwerk/test/moz.build
@@ -1,15 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['httpserver', 'browser', 'mochitests']
+TEST_TOOL_DIRS += ['httpserver', 'browser', 'mochitests']
 
 MODULE = 'test_necko'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
 
 # FIXME/bug 575918: out-of-process xpcshell is broken on OS X
 if CONFIG['OS_ARCH'] != 'Darwin':
     XPCSHELL_TESTS_MANIFESTS += ['unit_ipc/xpcshell.ini']
--- a/parser/htmlparser/tests/mochitest/html5lib_tree_construction/moz.build
+++ b/parser/htmlparser/tests/mochitest/html5lib_tree_construction/moz.build
@@ -1,7 +1,7 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['scripted']
+TEST_TOOL_DIRS += ['scripted']
--- a/parser/htmlparser/tests/mochitest/moz.build
+++ b/parser/htmlparser/tests/mochitest/moz.build
@@ -1,7 +1,7 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['dir_bug534293', 'html5lib_tree_construction']
+TEST_TOOL_DIRS += ['dir_bug534293', 'html5lib_tree_construction']
--- a/parser/htmlparser/tests/moz.build
+++ b/parser/htmlparser/tests/moz.build
@@ -1,7 +1,7 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['mochitest']
+TEST_TOOL_DIRS += ['mochitest']
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -390,16 +390,19 @@ class RecursiveMakeBackend(CommonBackend
 
         if len(obj.external_make_dirs):
             fh.write('DIRS += %s\n' % ' '.join(obj.external_make_dirs))
 
         if len(obj.parallel_external_make_dirs):
             fh.write('PARALLEL_DIRS += %s\n' %
                 ' '.join(obj.parallel_external_make_dirs))
 
+        if obj.is_tool_dir:
+            fh.write('IS_TOOL_DIR := 1\n')
+
     def _process_exports(self, obj, exports, backend_file, namespace=""):
         # This may not be needed, but is present for backwards compatibility
         # with the old make rules, just in case.
         if not obj.dist_install:
             return
 
         strings = exports.get_strings()
         if namespace:
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -185,16 +185,17 @@ class TreeMetadataEmitter(LoggingMixin):
         o = DirectoryTraversal(sandbox)
         o.dirs = sandbox.get('DIRS', [])
         o.parallel_dirs = sandbox.get('PARALLEL_DIRS', [])
         o.tool_dirs = sandbox.get('TOOL_DIRS', [])
         o.test_dirs = sandbox.get('TEST_DIRS', [])
         o.test_tool_dirs = sandbox.get('TEST_TOOL_DIRS', [])
         o.external_make_dirs = sandbox.get('EXTERNAL_MAKE_DIRS', [])
         o.parallel_external_make_dirs = sandbox.get('PARALLEL_EXTERNAL_MAKE_DIRS', [])
+        o.is_tool_dir = sandbox.get('IS_TOOL_DIR', False)
 
         if 'TIERS' in sandbox:
             for tier in sandbox['TIERS']:
                 o.tier_dirs[tier] = sandbox['TIERS'][tier]['regular'] + \
                     sandbox['TIERS'][tier]['external']
                 o.tier_static_dirs[tier] = sandbox['TIERS'][tier]['static']
 
         yield o
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -168,16 +168,20 @@ class MozbuildSandbox(Sandbox):
             d['TOPOBJDIR'] = topobjdir
             d['RELATIVEDIR'] = reldir
             d['SRCDIR'] = os.path.join(topsrcdir, reldir).replace(os.sep, '/').rstrip('/')
             d['OBJDIR'] = os.path.join(topobjdir, reldir).replace(os.sep, '/').rstrip('/')
 
             d['CONFIG'] = ReadOnlyDefaultDict(self.config.substs_unicode,
                 global_default=None)
 
+            var = metadata.get('var', None)
+            if var and var in ['TOOL_DIRS', 'TEST_TOOL_DIRS']:
+                d['IS_TOOL_DIR'] = True
+
             # Register functions.
             for name, func in FUNCTIONS.items():
                 d[name] = getattr(self, func[0])
 
     def exec_file(self, path, filesystem_absolute=False):
         """Override exec_file to normalize paths and restrict file loading.
 
         If the path is absolute, behavior is governed by filesystem_absolute.
@@ -327,17 +331,18 @@ class BuildReaderError(Exception):
             s.write('\n')
 
         if self.sandbox_error is not None:
             self._print_sandbox_error(s)
         elif self.validation_error is not None:
             s.write('The error occurred when validating the result of ')
             s.write('the execution. The reported error is:\n')
             s.write('\n')
-            s.write('    %s\n' % self.validation_error.message)
+            s.write(''.join('    %s\n' % l
+                            for l in self.validation_error.message.splitlines()))
             s.write('\n')
         else:
             s.write('The error appears to be part of the %s ' % __name__)
             s.write('Python module itself! It is possible you have stumbled ')
             s.write('across a legitimate bug.\n')
             s.write('\n')
 
             for l in traceback.format_exception(type(self.other), self.other,
@@ -629,16 +634,30 @@ class BuildReader(object):
             return
 
         self._read_files.add(path)
 
         time_start = time.time()
         sandbox = MozbuildSandbox(self.config, path, metadata=metadata)
         sandbox.exec_file(path, filesystem_absolute=filesystem_absolute)
         sandbox.execution_time = time.time() - time_start
+        var = metadata.get('var', None)
+        forbidden = {
+            'TOOL_DIRS': ['DIRS', 'PARALLEL_DIRS', 'TEST_DIRS'],
+            'TEST_TOOL_DIRS': ['DIRS', 'PARALLEL_DIRS', 'TEST_DIRS', 'TOOL_DIRS'],
+        }
+        if var in forbidden:
+            matches = [v for v in forbidden[var] if sandbox[v]]
+            if matches:
+                raise SandboxValidationError('%s is registered as %s in %s/moz.build.\n'
+                    'The %s variable%s not allowed in such directories.'
+                    % (sandbox['RELATIVEDIR'], var, metadata['parent'],
+                       ' and '.join(', '.join(matches).rsplit(', ', 1)),
+                       's are' if len(matches) > 1 else ' is'))
+
         yield sandbox
 
         # Traverse into referenced files.
 
         # We first collect directories populated in variables.
         dir_vars = ['DIRS', 'PARALLEL_DIRS', 'TOOL_DIRS']
 
         if self.config.substs.get('ENABLE_TESTS', False) == '1':
@@ -653,33 +672,37 @@ class BuildReader(object):
                 continue
 
             for d in sandbox[var]:
                 if d in recurse_info:
                     raise SandboxValidationError(
                         'Directory (%s) registered multiple times in %s' % (
                             d, var))
 
-                recurse_info[d] = {'tier': metadata.get('tier', None)}
+                recurse_info[d] = {'tier': metadata.get('tier', None),
+                                   'parent': sandbox['RELATIVEDIR'],
+                                   'var': var}
 
         # We also have tiers whose members are directories.
         if 'TIERS' in sandbox:
             if not read_tiers:
                 raise SandboxValidationError(
                     'TIERS defined but it should not be')
 
             for tier, values in sandbox['TIERS'].items():
                 # We don't descend into static directories because static by
                 # definition is external to the build system.
                 for d in values['regular']:
                     if d in recurse_info:
                         raise SandboxValidationError(
                             'Tier directory (%s) registered multiple '
                             'times in %s' % (d, tier))
-                    recurse_info[d] = {'tier': tier}
+                    recurse_info[d] = {'tier': tier,
+                                       'parent': sandbox['RELATIVEDIR'],
+                                       'var': 'DIRS'}
 
         curdir = os.path.dirname(path)
         for relpath, child_metadata in recurse_info.items():
             child_path = os.path.join(curdir, relpath, 'moz.build')
 
             # Ensure we don't break out of the topsrcdir. We don't do realpath
             # because it isn't necessary. If there are symlinks in the srcdir,
             # that's not our problem. We're not a hosted application: we don't
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/foo/moz.build
@@ -0,0 +1,2 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+DIRS = ['biz']
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/moz.build
@@ -0,0 +1,5 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+TOOL_DIRS = ['foo']
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -239,11 +239,19 @@ class TestBuildReader(unittest.TestCase)
 
         with self.assertRaises(BuildReaderError) as bre:
             list(reader.read_topsrcdir())
 
         e = bre.exception
         self.assertIn('A moz.build file called the error() function.', str(e))
         self.assertIn('    Some error.', str(e))
 
+    def test_error_traversal_tools(self):
+        reader = self.reader('reader-error-traversal-tools')
+
+        with self.assertRaises(BuildReaderError) as bre:
+            list(reader.read_topsrcdir())
+
+        e = bre.exception
+        self.assertIn('The DIRS variable is not allowed in such directories.', str(e))
 
 if __name__ == '__main__':
     main()
--- a/rdf/tests/moz.build
+++ b/rdf/tests/moz.build
@@ -1,11 +1,11 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += ['rdfcat', 'rdfpoll', 'triplescat']
+TEST_TOOL_DIRS += ['rdfcat', 'rdfpoll', 'triplescat']
 
 MODULE = 'test_rdf'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
--- a/toolkit/components/url-classifier/tests/moz.build
+++ b/toolkit/components/url-classifier/tests/moz.build
@@ -1,11 +1,11 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['mochitest']
+TEST_TOOL_DIRS += ['mochitest']
 
 MODULE = 'test_url-classifier'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
--- a/xpcom/sample/moz.build
+++ b/xpcom/sample/moz.build
@@ -1,15 +1,17 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += ['program']
+# If you're copying from this file, you'll likely need to replace
+# TEST_TOOL_DIRS with DIRS.
+TEST_TOOL_DIRS += ['program']
 
 # XPIDL_SOURCES specifies IDL files. The build system runs the xpidl tool
 # on these files to generate C++ headers and .xpt typelib files.
 XPIDL_SOURCES += ['nsISample.idl']
 
 # MODULE specifies where header files from this Makefile are installed,
 # i.e. dist/include/xpcomsample
 MODULE = 'xpcomsample'
--- a/xpcom/tests/moz.build
+++ b/xpcom/tests/moz.build
@@ -1,26 +1,26 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += [
+TEST_TOOL_DIRS += [
     'external',
     'component',
     'bug656331_component',
     'component_no_aslr',
 ]
 
 if CONFIG['OS_ARCH'] == 'WINNT':
-    DIRS += ['windows']
+    TEST_TOOL_DIRS += ['windows']
 
 if CONFIG['DEHYDRA_PATH']:
-    DIRS += ['static-checker']
+    TEST_TOOL_DIRS += ['static-checker']
 
 EXPORTS.testing += [
     'TestHarness.h',
 ]
 
 sources = [
     'TestArguments',
     'TestBlockingProcess',