Bug 784841 - Part 20: Fixes to support building external projects; r=glandium
authorGregory Szorc <gps@mozilla.com>
Thu, 28 Feb 2013 12:56:40 +0100
changeset 123345 614fb1e40f6cfb10344d5d0ea6e76032cf2db017
parent 123344 14ab81518d2b3b72ada9d6857a3de773f4f06f6e
child 123346 831a3b6a4e07ad2624e7685147efc1013d0c4290
push id23787
push userMs2ger@gmail.com
push dateThu, 28 Feb 2013 17:37:42 +0000
treeherdermozilla-inbound@c65d59d33aa8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs784841
milestone22.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 784841 - Part 20: Fixes to support building external projects; r=glandium This is extremely hacky. It will likely need many refinements as we figure out how we want external projects to be handled.
b2g/app.mozbuild
browser/app.mozbuild
configure.in
mobile/android/app.mozbuild
mobile/xul/app.mozbuild
python/mozbuild/mozbuild/backend/base.py
python/mozbuild/mozbuild/backend/configenvironment.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/reader.py
toolkit/toolkit.mozbuild
xulrunner/app.mozbuild
--- a/b2g/app.mozbuild
+++ b/b2g/app.mozbuild
@@ -1,15 +1,17 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 if not CONFIG['LIBXUL_SDK']:
     app_libxul_dirs = []
+    app_libxul_static_dirs = []
+
     include('/toolkit/toolkit.mozbuild')
 elif CONFIG['ENABLE_TESTS']:
     add_tier_dir('testharness', 'testing/mochitest')
 
 if CONFIG['MOZ_EXTENSIONS']:
     add_tier_dir('app', 'extensions')
 
 add_tier_dir('app', CONFIG['MOZ_BRANDING_DIRECTORY'])
--- a/browser/app.mozbuild
+++ b/browser/app.mozbuild
@@ -1,15 +1,16 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 if not CONFIG['LIBXUL_SDK']:
     app_libxul_dirs = []
+    app_libxul_static_dirs = []
     include('/toolkit/toolkit.mozbuild')
 
 if CONFIG['MOZ_EXTENSIONS']:
     add_tier_dir('app', 'extensions')
 
 add_tier_dir('app', [CONFIG['MOZ_BRANDING_DIRECTORY']])
 
 if CONFIG['MOZ_WEBAPP_RUNTIME']:
--- a/configure.in
+++ b/configure.in
@@ -4291,16 +4291,22 @@ case "${target}" in
         MOZ_THEME_FASTSTRIPE=1
         MOZ_TREE_FREETYPE=1
         MOZ_MEMORY=1
         MOZ_RAW=1
         ;;
 
 esac
 
+MOZ_ARG_WITH_STRING(external-source-dir,
+[  --with-external-source-dir=dir
+                          External directory containing additional build files.],
+[ EXTERNAL_SOURCE_DIR=$withval])
+AC_SUBST(EXTERNAL_SOURCE_DIR)
+
 MOZ_ARG_ENABLE_STRING(application,
 [  --enable-application=APP
                           Options include:
                             browser (Firefox)
                             xulrunner
                             tools/update-packaging (AUS-related packaging tools)],
 [ MOZ_BUILD_APP=$enableval ] )
 
--- a/mobile/android/app.mozbuild
+++ b/mobile/android/app.mozbuild
@@ -1,15 +1,16 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 if not CONFIG['LIBXUL_SDK']:
     app_libxul_dirs = ['mobile/android/components/build']
+    app_libxul_static_dirs = []
 
     include('/toolkit/toolkit.mozbuild')
 
 elif CONFIG['ENABLE_TESTS']:
     add_tier_dir('testharness', 'testing/mochitest')
 
 if CONFIG['MOZ_EXTENSIONS']:
     add_tier_dir('app', 'extensions')
--- a/mobile/xul/app.mozbuild
+++ b/mobile/xul/app.mozbuild
@@ -1,15 +1,16 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 if not CONFIG['LIBXUL_SDK']:
     app_libxul_dirs = ['mobile/xul/components/build']
+    app_libxul_static_dirs = []
     include('/toolkit/toolkit.mozbuild')
 
 elif CONFIG['ENABLE_TESTS']:
     add_tier_dir('testharness', 'testing/mochitest')
 
 if CONFIG['MOZ_EXTENSIONS']:
     add_tier_dir('app', 'extensions')
 
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -30,24 +30,45 @@ class BuildBackend(LoggingMixin):
 
     def __init__(self, environment):
         assert isinstance(environment, ConfigEnvironment)
 
         self.populate_logger()
 
         self.environment = environment
 
+        self._environments = {}
+        self._environments[environment.topobjdir] = environment
+
         self._init()
 
     def _init():
         """Hook point for child classes to perform actions during __init__.
 
         This exists so child classes don't need to implement __init__.
         """
 
+    def get_environment(self, obj):
+        """Obtain the ConfigEnvironment for a specific object.
+
+        This is used to support external source directories which operate in
+        their own topobjdir and have their own ConfigEnvironment.
+
+        This is somewhat hacky and should be considered for rewrite if external
+        project integration is rewritten.
+        """
+        environment = self._environments.get(obj.topobjdir, None)
+        if not environment:
+            config_status = os.path.join(obj.topobjdir, 'config.status')
+
+            environment = ConfigEnvironment.from_config_status(config_status)
+            self._environments[obj.topobjdir] = environment
+
+        return environment
+
     def consume(self, objs):
         """Consume a stream of TreeMetadata instances.
 
         This is the main method of the interface. This is what takes the
         frontend output and does something with it.
 
         Child classes are not expected to implement this method. Instead, the
         base class consumes objects and calls methods (possibly) implemented by
--- a/python/mozbuild/mozbuild/backend/configenvironment.py
+++ b/python/mozbuild/mozbuild/backend/configenvironment.py
@@ -20,16 +20,49 @@ from ..util import (
 RE_SHELL_ESCAPE = re.compile('''([ \t`#$^&*(){}\\|;'"<>?\[\]])''')
 
 
 def shell_escape(s):
     """Escape some characters with a backslash, and double dollar signs."""
     return RE_SHELL_ESCAPE.sub(r'\\\1', str(s)).replace('$', '$$')
 
 
+class BuildConfig(object):
+    """Represents the output of configure."""
+
+    def __init__(self):
+        self.topsrcdir = None
+        self.topobjdir = None
+        self.defines = {}
+        self.non_global_defines = []
+        self.substs = {}
+        self.files = []
+
+    @staticmethod
+    def from_config_status(path):
+        """Create an instance from a config.status file."""
+
+        with open(path, 'rt') as fh:
+            source = fh.read()
+            code = compile(source, path, 'exec', dont_inherit=1)
+            g = {
+                '__builtins__': __builtins__,
+                '__file__': path,
+            }
+            l = {}
+            exec(code, g, l)
+
+            config = BuildConfig()
+
+            for name in l['__all__']:
+                setattr(config, name, l[name])
+
+            return config
+
+
 class ConfigEnvironment(object):
     """Perform actions associated with a configured but bare objdir.
 
     The purpose of this class is to preprocess files from the source directory
     and output results in the object directory.
 
     There are two types of files: config files and config headers,
     each treated through a different member function.
@@ -75,16 +108,23 @@ class ConfigEnvironment(object):
             if not name in non_global_defines]
         self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name,
             shell_escape(self.defines[name])) for name in global_defines])
         self.substs['ALLSUBSTS'] = '\n'.join(sorted(['%s = %s' % (name,
             self.substs[name]) for name in self.substs]))
         self.substs['ALLDEFINES'] = '\n'.join(sorted(['#define %s %s' % (name,
             self.defines[name]) for name in global_defines]))
 
+    @staticmethod
+    def from_config_status(path):
+        config = BuildConfig.from_config_status(path)
+
+        return ConfigEnvironment(config.topsrcdir, config.topobjdir,
+            config.defines, config.non_global_defines, config.substs)
+
     def get_relative_srcdir(self, file):
         '''Returns the relative source directory for the given file, always
         using / as a path separator.
         '''
         assert(isinstance(file, basestring))
         dir = posixpath.dirname(relpath(file, self.topobjdir).replace(os.sep, '/'))
         if dir:
             return dir
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -54,19 +54,20 @@ class BackendMakeFile(object):
 
     We work around this problem by having backend generation update the mtime
     of backend.mk if they are older than their inputs - even if the file
     contents did not change. This is essentially a middle ground between
     always updating backend.mk and only updating the backend.mk that was out
     of date during recursion.
     """
 
-    def __init__(self, srcdir, objdir):
+    def __init__(self, srcdir, objdir, environment):
         self.srcdir = srcdir
         self.objdir = objdir
+        self.environment = environment
         self.path = os.path.join(objdir, 'backend.mk')
 
         # Filenames that influenced the content of this file.
         self.inputs = set()
 
         self.fh = FileAvoidWrite(self.path)
         self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
         self.fh.write('\n')
@@ -125,31 +126,31 @@ class RecursiveMakeBackend(BuildBackend)
 
     def _init(self):
         self._backend_files = {}
 
     def consume_object(self, obj):
         """Write out build files necessary to build with recursive make."""
 
         backend_file = self._backend_files.get(obj.srcdir,
-            BackendMakeFile(obj.srcdir, obj.objdir))
+            BackendMakeFile(obj.srcdir, obj.objdir, self.get_environment(obj)))
 
         # Define the paths that will trigger a backend rebuild. We always
         # add autoconf.mk because that is proxy for CONFIG. We can't use
         # config.status because there is no make target for that!
         autoconf_path = os.path.join(obj.topobjdir, 'config', 'autoconf.mk')
         backend_file.inputs.add(autoconf_path)
         backend_file.inputs |= obj.sandbox_all_paths
 
         if isinstance(obj, DirectoryTraversal):
             self._process_directory_traversal(obj, backend_file)
         elif isinstance(obj, ConfigFileSubstitution):
             backend_file.write('SUBSTITUTE_FILES += %s\n' % obj.relpath)
 
-            self.environment.create_config_file(obj.output_path)
+            backend_file.environment.create_config_file(obj.output_path)
 
         self._backend_files[obj.srcdir] = backend_file
 
     def consume_finished(self):
         for srcdir in sorted(self._backend_files.keys()):
             bf = self._backend_files[srcdir]
 
             if not os.path.exists(bf.objdir):
@@ -158,17 +159,17 @@ class RecursiveMakeBackend(BuildBackend)
             makefile_in = os.path.join(srcdir, 'Makefile.in')
 
             if not os.path.exists(makefile_in):
                 raise Exception('Could not find Makefile.in: %s' % makefile_in)
 
             out_path = os.path.join(bf.objdir, 'Makefile')
             self.log(logging.DEBUG, 'create_makefile', {'path': out_path},
                 'Generating makefile: {path}')
-            self.environment.create_config_file(out_path)
+            bf.environment.create_config_file(out_path)
 
             bf.write('SUBSTITUTE_FILES += Makefile\n')
             bf.close()
 
     def _process_directory_traversal(self, obj, backend_file):
         """Process a data.DirectoryTraversal instance."""
         fh = backend_file.fh
 
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -51,20 +51,52 @@ from .sandbox_symbols import (
 
 if sys.version_info.major == 2:
     text_type = unicode
     type_type = types.TypeType
 else:
     text_type = str
     type_type = type
 
+
 def log(logger, level, action, params, formatter):
     logger.log(level, formatter, extra={'action': action, 'params': params})
 
 
+def is_read_allowed(path, config):
+    """Whether we are allowed to load a mozbuild file at the specified path.
+
+    This is used as cheap security to ensure the build is isolated to known
+    source directories.
+
+    We are allowed to read from the main source directory and any defined
+    external source directories. The latter is to allow 3rd party applications
+    to hook into our build system.
+    """
+    assert os.path.isabs(path)
+    assert os.path.isabs(config.topsrcdir)
+
+    path = os.path.normpath(path)
+    topsrcdir = os.path.normpath(config.topsrcdir)
+
+    if path.startswith(topsrcdir):
+        return True
+
+    external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split()
+    for external in external_dirs:
+        if not os.path.isabs(external):
+            external = os.path.join(config.topsrcdir, external)
+        external = os.path.normpath(external)
+
+        if path.startswith(external):
+            return True
+
+    return False
+
+
 class SandboxCalledError(SandboxError):
     """Represents an error resulting from calling the error() function."""
 
     def __init__(self, file_stack, message):
         SandboxError.__init__(self, file_stack)
         self.message = message
 
 
@@ -82,29 +114,48 @@ class MozbuildSandbox(Sandbox):
         to compute encountered relative paths.
         """
         Sandbox.__init__(self, allowed_variables=VARIABLES)
 
         self.config = config
 
         topobjdir = os.path.abspath(config.topobjdir)
 
-        # This may not always hold true. If we ever have autogenerated mozbuild
-        # files in topobjdir, we'll need to change this.
-        assert os.path.normpath(path).startswith(os.path.normpath(config.topsrcdir))
-        assert not os.path.normpath(path).startswith(os.path.normpath(topobjdir))
+        topsrcdir = config.topsrcdir
+        if not path.startswith(topsrcdir):
+            external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split()
+            for external in external_dirs:
+                if not os.path.isabs(external):
+                    external = os.path.join(config.topsrcdir, external)
+
+                external = os.path.normpath(external)
+
+                if not path.startswith(external):
+                    continue
 
-        relpath = os.path.relpath(path, config.topsrcdir).replace(os.sep, '/')
+                topsrcdir = external
+
+                # This is really hacky and should be replaced with something
+                # more robust. We assume that if an external source directory
+                # is in play that the main build system is built in a
+                # subdirectory of its topobjdir. Therefore, the topobjdir of
+                # the external source directory is the parent of our topobjdir.
+                topobjdir = os.path.dirname(topobjdir)
+                break
+
+        self.topsrcdir = topsrcdir
+
+        relpath = os.path.relpath(path, topsrcdir).replace(os.sep, '/')
         reldir = os.path.dirname(relpath)
 
         with self._globals.allow_all_writes() as d:
-            d['TOPSRCDIR'] = config.topsrcdir
+            d['TOPSRCDIR'] = topsrcdir
             d['TOPOBJDIR'] = topobjdir
             d['RELATIVEDIR'] = reldir
-            d['SRCDIR'] = os.path.join(config.topsrcdir, reldir).replace(os.sep, '/').rstrip('/')
+            d['SRCDIR'] = os.path.join(topsrcdir, reldir).replace(os.sep, '/').rstrip('/')
             d['OBJDIR'] = os.path.join(topobjdir, reldir).replace(os.sep, '/').rstrip('/')
 
             # config.status does not yet use unicode. However, mozbuild expects
             # unicode everywhere. So, decode binary into unicode as necessary.
             # Bug 844509 tracks a better way to do this.
             substs = {}
             for k, v in config.substs.items():
                 if not isinstance(v, text_type):
@@ -113,50 +164,48 @@ class MozbuildSandbox(Sandbox):
                 substs[k] = v
 
             d['CONFIG'] = ReadOnlyDefaultDict(substs, global_default=None)
 
             # Register functions.
             for name, func in FUNCTIONS.items():
                 d[name] = getattr(self, func[0])
 
-        self._normalized_topsrcdir = os.path.normpath(config.topsrcdir)
-
     def exec_file(self, path, filesystem_absolute=False):
         """Override exec_file to normalize paths and restrict file loading.
 
         If the path is absolute, behavior is governed by filesystem_absolute.
         If filesystem_absolute is True, the path is interpreted as absolute on
         the actual filesystem. If it is false, the path is treated as absolute
         within the current topsrcdir.
 
         If the path is not absolute, it will be treated as relative to the
         currently executing file. If there is no currently executing file, it
         will be treated as relative to topsrcdir.
 
         Paths will be rejected if they do not fall under topsrcdir.
         """
         if os.path.isabs(path):
             if not filesystem_absolute:
-                path = os.path.normpath(os.path.join(self.config.topsrcdir,
+                path = os.path.normpath(os.path.join(self.topsrcdir,
                     path[1:]))
 
         else:
             if len(self._execution_stack):
                 path = os.path.normpath(os.path.join(
                     os.path.dirname(self._execution_stack[-1]),
                     path))
             else:
                 path = os.path.normpath(os.path.join(
-                    self.config.topsrcdir, path))
+                    self.topsrcdir, path))
 
         # realpath() is needed for true security. But, this isn't for security
         # protection, so it is omitted.
         normalized_path = os.path.normpath(path)
-        if not normalized_path.startswith(self._normalized_topsrcdir):
+        if not is_read_allowed(normalized_path, self.config):
             raise SandboxLoadError(list(self._execution_stack),
                 sys.exc_info()[2], illegal_path=path)
 
         Sandbox.exec_file(self, path)
 
     def _add_tier_directory(self, tier, reldir, static=False):
         """Register a tier directory with the build."""
         if isinstance(reldir, text_type):
@@ -479,17 +528,16 @@ class BuildReader(object):
     """
 
     def __init__(self, config):
         self.config = config
         self.topsrcdir = config.topsrcdir
 
         self._log = logging.getLogger(__name__)
         self._read_files = set()
-        self._normalized_topsrcdir = os.path.normpath(config.topsrcdir)
         self._execution_stack = []
 
     def read_topsrcdir(self):
         """Read the tree of mozconfig files into a data structure.
 
         This starts with the tree's top-most mozbuild file and descends into
         all linked mozbuild files until all relevant files have been evaluated.
 
@@ -610,19 +658,19 @@ class BuildReader(object):
         for relpath in dirs:
             child_path = os.path.join(curdir, relpath, 'moz.build')
 
             # Ensure we don't break out of the topsrcdir. We don't do realpath
             # because it isn't necessary. If there are symlinks in the srcdir,
             # that's not our problem. We're not a hosted application: we don't
             # need to worry about security too much.
             child_path = os.path.normpath(child_path)
-            if not child_path.startswith(self._normalized_topsrcdir):
+            if not is_read_allowed(child_path, self.config):
                 raise SandboxValidationError(
-                    'Attempting to process file outside of topsrcdir: %s' %
+                    'Attempting to process file outside of allowed paths: %s' %
                         child_path)
 
             if not descend:
                 continue
 
             for res in self.read_mozbuild(child_path, read_tiers=False,
                 filesystem_absolute=True):
                 yield res
--- a/toolkit/toolkit.mozbuild
+++ b/toolkit/toolkit.mozbuild
@@ -204,16 +204,17 @@ add_tier_dir('platform', 'js/ductwork/de
 add_tier_dir('platform', 'other-licenses/snappy')
 
 if CONFIG['MOZ_GIO_COMPONENT']:
     add_tier_dir('platform', 'extensions/gio')
 
 # Applications can cheat and ask for code to be
 # built before libxul so it can be linked into libxul.
 add_tier_dir('platform', app_libxul_dirs)
+add_tier_dir('platform', app_libxul_static_dirs, static=True)
 
 add_tier_dir('platform', 'toolkit/library')
 add_tier_dir('platform', 'xpcom/stub')
 
 if CONFIG['MOZ_REPLACE_MALLOC']:
     add_tier_dir('platform', 'memory/replace')
 
 if CONFIG['NS_TRACE_MALLOC']:
--- a/xulrunner/app.mozbuild
+++ b/xulrunner/app.mozbuild
@@ -1,14 +1,15 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 app_libxul_dirs = []
+app_libxul_static_dirs = []
 include('/toolkit/toolkit.mozbuild')
 
 if CONFIG['MOZ_EXTENSIONS']:
     add_tier_dir('app', 'extensions')
 
 if CONFIG['OS_ARCH'] == 'WINNT' and (CONFIG['ENABLE_TESTS'] or
         CONFIG['MOZILLA_OFFICIAL']):
     add_tier_dir('app', 'embedding/tests/winEmbed')