Bug 917086 - Disallow DIRS, PARALLEL_DIRS and TEST_DIRS under TOOL_DIRS and TEST_TOOL_DIRS, and adapt moz.build files accordingly. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 19 Sep 2013 07:43:02 +0900
changeset 147813 8dd27d839acc22b5303b366afb9a3968e5e219fc
parent 147812 fdaa112654646d6f63e31216b68c29fe55e13487
child 147814 554bfe767519cb699b62a73cba8cd7d95b3bfc7f
push id25317
push useremorley@mozilla.com
push dateThu, 19 Sep 2013 14:37:38 +0000
treeherderautoland@70a765607344 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs917086
milestone27.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 917086 - Disallow DIRS, PARALLEL_DIRS and TEST_DIRS under TOOL_DIRS and TEST_TOOL_DIRS, and adapt moz.build files accordingly. r=gps Also mark TOOL_DIRS/TEST_TOOL_DIRS directories in backend.mk and recurse them normally instead of forcing make -C dir libs for them.
config/recurse.mk
config/rules.mk
content/base/test/moz.build
js/src/config/recurse.mk
js/src/config/rules.mk
js/xpconnect/tests/moz.build
layout/style/test/moz.build
modules/libjar/test/moz.build
netwerk/test/moz.build
parser/htmlparser/tests/mochitest/html5lib_tree_construction/moz.build
parser/htmlparser/tests/mochitest/moz.build
parser/htmlparser/tests/moz.build
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/foo/biz/moz.build
python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/foo/moz.build
python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/moz.build
python/mozbuild/mozbuild/test/frontend/test_reader.py
rdf/tests/moz.build
toolkit/components/url-classifier/tests/moz.build
xpcom/sample/moz.build
xpcom/tests/moz.build
--- a/config/recurse.mk
+++ b/config/recurse.mk
@@ -38,15 +38,12 @@ ifdef PARALLEL_DIRS
 	+@$(MAKE) $$(PARALLEL_DIRS_$(1))
 endif
 	$$(LOOP_OVER_DIRS)
 
 endef
 
 $(foreach subtier,export compile libs tools,$(eval $(call CREATE_SUBTIER_TRAVERSAL_RULE,$(subtier))))
 
-compile export:: $(SUBMAKEFILES)
+compile export tools:: $(SUBMAKEFILES)
 	$(LOOP_OVER_TOOL_DIRS)
 
-tools:: $(SUBMAKEFILES)
-	$(foreach dir,$(TOOL_DIRS),$(call SUBMAKE,libs,$(dir)))
-
 endif
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -732,16 +732,23 @@ ifndef NO_MAKEFILE_RULE
 GLOBAL_DEPS += Makefile.in
 endif
 
 ##############################################
 compile:: $(OBJS) $(HOST_OBJS)
 
 include $(topsrcdir)/config/makefiles/target_libs.mk
 
+ifdef IS_TOOL_DIR
+# One would think "tools:: libs" would work, but it turns out that combined with
+# bug 907365, this makes make forget to run some rules sometimes.
+tools::
+	@$(MAKE) libs
+endif
+
 ##############################################
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 ifdef MOZ_PROFILE_USE
 ifeq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 # When building with PGO, we have to make sure to re-link
 # in the MOZ_PROFILE_USE phase if we linked in the
 # MOZ_PROFILE_GENERATE phase. We'll touch this pgo.relink
 # file in the link rule in the GENERATE phase to indicate
--- a/content/base/test/moz.build
+++ b/content/base/test/moz.build
@@ -1,15 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += [
+TEST_TOOL_DIRS += [
     'chrome',
     'csp',
     'websocket_hybi',
 ]
 
 MODULE = 'content'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
--- a/js/src/config/recurse.mk
+++ b/js/src/config/recurse.mk
@@ -38,15 +38,12 @@ ifdef PARALLEL_DIRS
 	+@$(MAKE) $$(PARALLEL_DIRS_$(1))
 endif
 	$$(LOOP_OVER_DIRS)
 
 endef
 
 $(foreach subtier,export compile libs tools,$(eval $(call CREATE_SUBTIER_TRAVERSAL_RULE,$(subtier))))
 
-compile export:: $(SUBMAKEFILES)
+compile export tools:: $(SUBMAKEFILES)
 	$(LOOP_OVER_TOOL_DIRS)
 
-tools:: $(SUBMAKEFILES)
-	$(foreach dir,$(TOOL_DIRS),$(call SUBMAKE,libs,$(dir)))
-
 endif
--- a/js/src/config/rules.mk
+++ b/js/src/config/rules.mk
@@ -732,16 +732,23 @@ ifndef NO_MAKEFILE_RULE
 GLOBAL_DEPS += Makefile.in
 endif
 
 ##############################################
 compile:: $(OBJS) $(HOST_OBJS)
 
 include $(topsrcdir)/config/makefiles/target_libs.mk
 
+ifdef IS_TOOL_DIR
+# One would think "tools:: libs" would work, but it turns out that combined with
+# bug 907365, this makes make forget to run some rules sometimes.
+tools::
+	@$(MAKE) libs
+endif
+
 ##############################################
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 ifdef MOZ_PROFILE_USE
 ifeq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 # When building with PGO, we have to make sure to re-link
 # in the MOZ_PROFILE_USE phase if we linked in the
 # MOZ_PROFILE_GENERATE phase. We'll touch this pgo.relink
 # file in the link rule in the GENERATE phase to indicate
--- a/js/xpconnect/tests/moz.build
+++ b/js/xpconnect/tests/moz.build
@@ -1,15 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += [
+TEST_TOOL_DIRS += [
     'idl',
     'mochitest',
     'chrome',
     'browser',
     'components/native',
     'components/js',
 ]
 
--- a/layout/style/test/moz.build
+++ b/layout/style/test/moz.build
@@ -1,13 +1,13 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['chrome']
+TEST_TOOL_DIRS += ['chrome']
 
 MODULE = 'layout'
 
 HOST_CPPSRCS += [
     'ListCSSProperties.cpp',
 ]
--- a/modules/libjar/test/moz.build
+++ b/modules/libjar/test/moz.build
@@ -1,11 +1,11 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += ['mochitest', 'chrome']
+TEST_TOOL_DIRS += ['mochitest', 'chrome']
 
 MODULE = 'test_libjar'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
--- a/netwerk/test/moz.build
+++ b/netwerk/test/moz.build
@@ -1,15 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['httpserver', 'browser', 'mochitests']
+TEST_TOOL_DIRS += ['httpserver', 'browser', 'mochitests']
 
 MODULE = 'test_necko'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
 
 # FIXME/bug 575918: out-of-process xpcshell is broken on OS X
 if CONFIG['OS_ARCH'] != 'Darwin':
     XPCSHELL_TESTS_MANIFESTS += ['unit_ipc/xpcshell.ini']
--- a/parser/htmlparser/tests/mochitest/html5lib_tree_construction/moz.build
+++ b/parser/htmlparser/tests/mochitest/html5lib_tree_construction/moz.build
@@ -1,7 +1,7 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['scripted']
+TEST_TOOL_DIRS += ['scripted']
--- a/parser/htmlparser/tests/mochitest/moz.build
+++ b/parser/htmlparser/tests/mochitest/moz.build
@@ -1,7 +1,7 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['dir_bug534293', 'html5lib_tree_construction']
+TEST_TOOL_DIRS += ['dir_bug534293', 'html5lib_tree_construction']
--- a/parser/htmlparser/tests/moz.build
+++ b/parser/htmlparser/tests/moz.build
@@ -1,7 +1,7 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['mochitest']
+TEST_TOOL_DIRS += ['mochitest']
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -390,16 +390,19 @@ class RecursiveMakeBackend(CommonBackend
 
         if len(obj.external_make_dirs):
             fh.write('DIRS += %s\n' % ' '.join(obj.external_make_dirs))
 
         if len(obj.parallel_external_make_dirs):
             fh.write('PARALLEL_DIRS += %s\n' %
                 ' '.join(obj.parallel_external_make_dirs))
 
+        if obj.is_tool_dir:
+            fh.write('IS_TOOL_DIR := 1\n')
+
     def _process_exports(self, obj, exports, backend_file, namespace=""):
         # This may not be needed, but is present for backwards compatibility
         # with the old make rules, just in case.
         if not obj.dist_install:
             return
 
         strings = exports.get_strings()
         if namespace:
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -185,16 +185,17 @@ class TreeMetadataEmitter(LoggingMixin):
         o = DirectoryTraversal(sandbox)
         o.dirs = sandbox.get('DIRS', [])
         o.parallel_dirs = sandbox.get('PARALLEL_DIRS', [])
         o.tool_dirs = sandbox.get('TOOL_DIRS', [])
         o.test_dirs = sandbox.get('TEST_DIRS', [])
         o.test_tool_dirs = sandbox.get('TEST_TOOL_DIRS', [])
         o.external_make_dirs = sandbox.get('EXTERNAL_MAKE_DIRS', [])
         o.parallel_external_make_dirs = sandbox.get('PARALLEL_EXTERNAL_MAKE_DIRS', [])
+        o.is_tool_dir = sandbox.get('IS_TOOL_DIR', False)
 
         if 'TIERS' in sandbox:
             for tier in sandbox['TIERS']:
                 o.tier_dirs[tier] = sandbox['TIERS'][tier]['regular'] + \
                     sandbox['TIERS'][tier]['external']
                 o.tier_static_dirs[tier] = sandbox['TIERS'][tier]['static']
 
         yield o
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -168,16 +168,20 @@ class MozbuildSandbox(Sandbox):
             d['TOPOBJDIR'] = topobjdir
             d['RELATIVEDIR'] = reldir
             d['SRCDIR'] = os.path.join(topsrcdir, reldir).replace(os.sep, '/').rstrip('/')
             d['OBJDIR'] = os.path.join(topobjdir, reldir).replace(os.sep, '/').rstrip('/')
 
             d['CONFIG'] = ReadOnlyDefaultDict(self.config.substs_unicode,
                 global_default=None)
 
+            var = metadata.get('var', None)
+            if var and var in ['TOOL_DIRS', 'TEST_TOOL_DIRS']:
+                d['IS_TOOL_DIR'] = True
+
             # Register functions.
             for name, func in FUNCTIONS.items():
                 d[name] = getattr(self, func[0])
 
     def exec_file(self, path, filesystem_absolute=False):
         """Override exec_file to normalize paths and restrict file loading.
 
         If the path is absolute, behavior is governed by filesystem_absolute.
@@ -327,17 +331,18 @@ class BuildReaderError(Exception):
             s.write('\n')
 
         if self.sandbox_error is not None:
             self._print_sandbox_error(s)
         elif self.validation_error is not None:
             s.write('The error occurred when validating the result of ')
             s.write('the execution. The reported error is:\n')
             s.write('\n')
-            s.write('    %s\n' % self.validation_error.message)
+            s.write(''.join('    %s\n' % l
+                            for l in self.validation_error.message.splitlines()))
             s.write('\n')
         else:
             s.write('The error appears to be part of the %s ' % __name__)
             s.write('Python module itself! It is possible you have stumbled ')
             s.write('across a legitimate bug.\n')
             s.write('\n')
 
             for l in traceback.format_exception(type(self.other), self.other,
@@ -629,16 +634,30 @@ class BuildReader(object):
             return
 
         self._read_files.add(path)
 
         time_start = time.time()
         sandbox = MozbuildSandbox(self.config, path, metadata=metadata)
         sandbox.exec_file(path, filesystem_absolute=filesystem_absolute)
         sandbox.execution_time = time.time() - time_start
+        var = metadata.get('var', None)
+        forbidden = {
+            'TOOL_DIRS': ['DIRS', 'PARALLEL_DIRS', 'TEST_DIRS'],
+            'TEST_TOOL_DIRS': ['DIRS', 'PARALLEL_DIRS', 'TEST_DIRS', 'TOOL_DIRS'],
+        }
+        if var in forbidden:
+            matches = [v for v in forbidden[var] if sandbox[v]]
+            if matches:
+                raise SandboxValidationError('%s is registered as %s in %s/moz.build.\n'
+                    'The %s variable%s not allowed in such directories.'
+                    % (sandbox['RELATIVEDIR'], var, metadata['parent'],
+                       ' and '.join(', '.join(matches).rsplit(', ', 1)),
+                       's are' if len(matches) > 1 else ' is'))
+
         yield sandbox
 
         # Traverse into referenced files.
 
         # We first collect directories populated in variables.
         dir_vars = ['DIRS', 'PARALLEL_DIRS', 'TOOL_DIRS']
 
         if self.config.substs.get('ENABLE_TESTS', False) == '1':
@@ -653,33 +672,37 @@ class BuildReader(object):
                 continue
 
             for d in sandbox[var]:
                 if d in recurse_info:
                     raise SandboxValidationError(
                         'Directory (%s) registered multiple times in %s' % (
                             d, var))
 
-                recurse_info[d] = {'tier': metadata.get('tier', None)}
+                recurse_info[d] = {'tier': metadata.get('tier', None),
+                                   'parent': sandbox['RELATIVEDIR'],
+                                   'var': var}
 
         # We also have tiers whose members are directories.
         if 'TIERS' in sandbox:
             if not read_tiers:
                 raise SandboxValidationError(
                     'TIERS defined but it should not be')
 
             for tier, values in sandbox['TIERS'].items():
                 # We don't descend into static directories because static by
                 # definition is external to the build system.
                 for d in values['regular']:
                     if d in recurse_info:
                         raise SandboxValidationError(
                             'Tier directory (%s) registered multiple '
                             'times in %s' % (d, tier))
-                    recurse_info[d] = {'tier': tier}
+                    recurse_info[d] = {'tier': tier,
+                                       'parent': sandbox['RELATIVEDIR'],
+                                       'var': 'DIRS'}
 
         curdir = os.path.dirname(path)
         for relpath, child_metadata in recurse_info.items():
             child_path = os.path.join(curdir, relpath, 'moz.build')
 
             # Ensure we don't break out of the topsrcdir. We don't do realpath
             # because it isn't necessary. If there are symlinks in the srcdir,
             # that's not our problem. We're not a hosted application: we don't
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/foo/moz.build
@@ -0,0 +1,2 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+DIRS = ['biz']
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-traversal-tools/moz.build
@@ -0,0 +1,5 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+TOOL_DIRS = ['foo']
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -239,11 +239,19 @@ class TestBuildReader(unittest.TestCase)
 
         with self.assertRaises(BuildReaderError) as bre:
             list(reader.read_topsrcdir())
 
         e = bre.exception
         self.assertIn('A moz.build file called the error() function.', str(e))
         self.assertIn('    Some error.', str(e))
 
+    def test_error_traversal_tools(self):
+        reader = self.reader('reader-error-traversal-tools')
+
+        with self.assertRaises(BuildReaderError) as bre:
+            list(reader.read_topsrcdir())
+
+        e = bre.exception
+        self.assertIn('The DIRS variable is not allowed in such directories.', str(e))
 
 if __name__ == '__main__':
     main()
--- a/rdf/tests/moz.build
+++ b/rdf/tests/moz.build
@@ -1,11 +1,11 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += ['rdfcat', 'rdfpoll', 'triplescat']
+TEST_TOOL_DIRS += ['rdfcat', 'rdfpoll', 'triplescat']
 
 MODULE = 'test_rdf'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
--- a/toolkit/components/url-classifier/tests/moz.build
+++ b/toolkit/components/url-classifier/tests/moz.build
@@ -1,11 +1,11 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-TEST_DIRS += ['mochitest']
+TEST_TOOL_DIRS += ['mochitest']
 
 MODULE = 'test_url-classifier'
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
--- a/xpcom/sample/moz.build
+++ b/xpcom/sample/moz.build
@@ -1,15 +1,17 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += ['program']
+# If you're copying from this file, you'll likely need to replace
+# TEST_TOOL_DIRS with DIRS.
+TEST_TOOL_DIRS += ['program']
 
 # XPIDL_SOURCES specifies IDL files. The build system runs the xpidl tool
 # on these files to generate C++ headers and .xpt typelib files.
 XPIDL_SOURCES += ['nsISample.idl']
 
 # MODULE specifies where header files from this Makefile are installed,
 # i.e. dist/include/xpcomsample
 MODULE = 'xpcomsample'
--- a/xpcom/tests/moz.build
+++ b/xpcom/tests/moz.build
@@ -1,26 +1,26 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += [
+TEST_TOOL_DIRS += [
     'external',
     'component',
     'bug656331_component',
     'component_no_aslr',
 ]
 
 if CONFIG['OS_ARCH'] == 'WINNT':
-    DIRS += ['windows']
+    TEST_TOOL_DIRS += ['windows']
 
 if CONFIG['DEHYDRA_PATH']:
-    DIRS += ['static-checker']
+    TEST_TOOL_DIRS += ['static-checker']
 
 EXPORTS.testing += [
     'TestHarness.h',
 ]
 
 sources = [
     'TestArguments',
     'TestBlockingProcess',