author | Gregory Szorc <gps@mozilla.com> |
Thu, 28 Feb 2013 12:56:40 +0100 | |
changeset 123269 | 614fb1e40f6cfb10344d5d0ea6e76032cf2db017 |
parent 123268 | 14ab81518d2b3b72ada9d6857a3de773f4f06f6e |
child 123346 | 831a3b6a4e07ad2624e7685147efc1013d0c4290 |
push id | 24376 |
push user | Ms2ger@gmail.com |
push date | Thu, 28 Feb 2013 17:05:14 +0000 |
treeherder | mozilla-central@c65d59d33aa8 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | glandium |
bugs | 784841 |
milestone | 22.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/b2g/app.mozbuild +++ b/b2g/app.mozbuild @@ -1,15 +1,17 @@ # vim: set filetype=python: # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. if not CONFIG['LIBXUL_SDK']: app_libxul_dirs = [] + app_libxul_static_dirs = [] + include('/toolkit/toolkit.mozbuild') elif CONFIG['ENABLE_TESTS']: add_tier_dir('testharness', 'testing/mochitest') if CONFIG['MOZ_EXTENSIONS']: add_tier_dir('app', 'extensions') add_tier_dir('app', CONFIG['MOZ_BRANDING_DIRECTORY'])
--- a/browser/app.mozbuild +++ b/browser/app.mozbuild @@ -1,15 +1,16 @@ # vim: set filetype=python: # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. if not CONFIG['LIBXUL_SDK']: app_libxul_dirs = [] + app_libxul_static_dirs = [] include('/toolkit/toolkit.mozbuild') if CONFIG['MOZ_EXTENSIONS']: add_tier_dir('app', 'extensions') add_tier_dir('app', [CONFIG['MOZ_BRANDING_DIRECTORY']]) if CONFIG['MOZ_WEBAPP_RUNTIME']:
--- a/configure.in +++ b/configure.in @@ -4291,16 +4291,22 @@ case "${target}" in MOZ_THEME_FASTSTRIPE=1 MOZ_TREE_FREETYPE=1 MOZ_MEMORY=1 MOZ_RAW=1 ;; esac +MOZ_ARG_WITH_STRING(external-source-dir, +[ --with-external-source-dir=dir + External directory containing additional build files.], +[ EXTERNAL_SOURCE_DIR=$withval]) +AC_SUBST(EXTERNAL_SOURCE_DIR) + MOZ_ARG_ENABLE_STRING(application, [ --enable-application=APP Options include: browser (Firefox) xulrunner tools/update-packaging (AUS-related packaging tools)], [ MOZ_BUILD_APP=$enableval ] )
--- a/mobile/android/app.mozbuild +++ b/mobile/android/app.mozbuild @@ -1,15 +1,16 @@ # vim: set filetype=python: # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. if not CONFIG['LIBXUL_SDK']: app_libxul_dirs = ['mobile/android/components/build'] + app_libxul_static_dirs = [] include('/toolkit/toolkit.mozbuild') elif CONFIG['ENABLE_TESTS']: add_tier_dir('testharness', 'testing/mochitest') if CONFIG['MOZ_EXTENSIONS']: add_tier_dir('app', 'extensions')
--- a/mobile/xul/app.mozbuild +++ b/mobile/xul/app.mozbuild @@ -1,15 +1,16 @@ # vim: set filetype=python: # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. if not CONFIG['LIBXUL_SDK']: app_libxul_dirs = ['mobile/xul/components/build'] + app_libxul_static_dirs = [] include('/toolkit/toolkit.mozbuild') elif CONFIG['ENABLE_TESTS']: add_tier_dir('testharness', 'testing/mochitest') if CONFIG['MOZ_EXTENSIONS']: add_tier_dir('app', 'extensions')
--- a/python/mozbuild/mozbuild/backend/base.py +++ b/python/mozbuild/mozbuild/backend/base.py @@ -30,24 +30,45 @@ class BuildBackend(LoggingMixin): def __init__(self, environment): assert isinstance(environment, ConfigEnvironment) self.populate_logger() self.environment = environment + self._environments = {} + self._environments[environment.topobjdir] = environment + self._init() def _init(): """Hook point for child classes to perform actions during __init__. This exists so child classes don't need to implement __init__. """ + def get_environment(self, obj): + """Obtain the ConfigEnvironment for a specific object. + + This is used to support external source directories which operate in + their own topobjdir and have their own ConfigEnvironment. + + This is somewhat hacky and should be considered for rewrite if external + project integration is rewritten. + """ + environment = self._environments.get(obj.topobjdir, None) + if not environment: + config_status = os.path.join(obj.topobjdir, 'config.status') + + environment = ConfigEnvironment.from_config_status(config_status) + self._environments[obj.topobjdir] = environment + + return environment + def consume(self, objs): """Consume a stream of TreeMetadata instances. This is the main method of the interface. This is what takes the frontend output and does something with it. Child classes are not expected to implement this method. Instead, the base class consumes objects and calls methods (possibly) implemented by
--- a/python/mozbuild/mozbuild/backend/configenvironment.py +++ b/python/mozbuild/mozbuild/backend/configenvironment.py @@ -20,16 +20,49 @@ from ..util import ( RE_SHELL_ESCAPE = re.compile('''([ \t`#$^&*(){}\\|;'"<>?\[\]])''') def shell_escape(s): """Escape some characters with a backslash, and double dollar signs.""" return RE_SHELL_ESCAPE.sub(r'\\\1', str(s)).replace('$', '$$') +class BuildConfig(object): + """Represents the output of configure.""" + + def __init__(self): + self.topsrcdir = None + self.topobjdir = None + self.defines = {} + self.non_global_defines = [] + self.substs = {} + self.files = [] + + @staticmethod + def from_config_status(path): + """Create an instance from a config.status file.""" + + with open(path, 'rt') as fh: + source = fh.read() + code = compile(source, path, 'exec', dont_inherit=1) + g = { + '__builtins__': __builtins__, + '__file__': path, + } + l = {} + exec(code, g, l) + + config = BuildConfig() + + for name in l['__all__']: + setattr(config, name, l[name]) + + return config + + class ConfigEnvironment(object): """Perform actions associated with a configured but bare objdir. The purpose of this class is to preprocess files from the source directory and output results in the object directory. There are two types of files: config files and config headers, each treated through a different member function. @@ -75,16 +108,23 @@ class ConfigEnvironment(object): if not name in non_global_defines] self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name, shell_escape(self.defines[name])) for name in global_defines]) self.substs['ALLSUBSTS'] = '\n'.join(sorted(['%s = %s' % (name, self.substs[name]) for name in self.substs])) self.substs['ALLDEFINES'] = '\n'.join(sorted(['#define %s %s' % (name, self.defines[name]) for name in global_defines])) + @staticmethod + def from_config_status(path): + config = BuildConfig.from_config_status(path) + + return ConfigEnvironment(config.topsrcdir, config.topobjdir, + config.defines, config.non_global_defines, config.substs) + def get_relative_srcdir(self, file): '''Returns the relative source directory for the given file, always using / as a path separator. ''' assert(isinstance(file, basestring)) dir = posixpath.dirname(relpath(file, self.topobjdir).replace(os.sep, '/')) if dir: return dir
--- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -54,19 +54,20 @@ class BackendMakeFile(object): We work around this problem by having backend generation update the mtime of backend.mk if they are older than their inputs - even if the file contents did not change. This is essentially a middle ground between always updating backend.mk and only updating the backend.mk that was out of date during recursion. """ - def __init__(self, srcdir, objdir): + def __init__(self, srcdir, objdir, environment): self.srcdir = srcdir self.objdir = objdir + self.environment = environment self.path = os.path.join(objdir, 'backend.mk') # Filenames that influenced the content of this file. self.inputs = set() self.fh = FileAvoidWrite(self.path) self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n') self.fh.write('\n') @@ -125,31 +126,31 @@ class RecursiveMakeBackend(BuildBackend) def _init(self): self._backend_files = {} def consume_object(self, obj): """Write out build files necessary to build with recursive make.""" backend_file = self._backend_files.get(obj.srcdir, - BackendMakeFile(obj.srcdir, obj.objdir)) + BackendMakeFile(obj.srcdir, obj.objdir, self.get_environment(obj))) # Define the paths that will trigger a backend rebuild. We always # add autoconf.mk because that is proxy for CONFIG. We can't use # config.status because there is no make target for that! autoconf_path = os.path.join(obj.topobjdir, 'config', 'autoconf.mk') backend_file.inputs.add(autoconf_path) backend_file.inputs |= obj.sandbox_all_paths if isinstance(obj, DirectoryTraversal): self._process_directory_traversal(obj, backend_file) elif isinstance(obj, ConfigFileSubstitution): backend_file.write('SUBSTITUTE_FILES += %s\n' % obj.relpath) - self.environment.create_config_file(obj.output_path) + backend_file.environment.create_config_file(obj.output_path) self._backend_files[obj.srcdir] = backend_file def consume_finished(self): for srcdir in sorted(self._backend_files.keys()): bf = self._backend_files[srcdir] if not os.path.exists(bf.objdir): @@ -158,17 +159,17 @@ class RecursiveMakeBackend(BuildBackend) makefile_in = os.path.join(srcdir, 'Makefile.in') if not os.path.exists(makefile_in): raise Exception('Could not find Makefile.in: %s' % makefile_in) out_path = os.path.join(bf.objdir, 'Makefile') self.log(logging.DEBUG, 'create_makefile', {'path': out_path}, 'Generating makefile: {path}') - self.environment.create_config_file(out_path) + bf.environment.create_config_file(out_path) bf.write('SUBSTITUTE_FILES += Makefile\n') bf.close() def _process_directory_traversal(self, obj, backend_file): """Process a data.DirectoryTraversal instance.""" fh = backend_file.fh
--- a/python/mozbuild/mozbuild/frontend/reader.py +++ b/python/mozbuild/mozbuild/frontend/reader.py @@ -51,20 +51,52 @@ from .sandbox_symbols import ( if sys.version_info.major == 2: text_type = unicode type_type = types.TypeType else: text_type = str type_type = type + def log(logger, level, action, params, formatter): logger.log(level, formatter, extra={'action': action, 'params': params}) +def is_read_allowed(path, config): + """Whether we are allowed to load a mozbuild file at the specified path. + + This is used as cheap security to ensure the build is isolated to known + source directories. + + We are allowed to read from the main source directory and any defined + external source directories. The latter is to allow 3rd party applications + to hook into our build system. + """ + assert os.path.isabs(path) + assert os.path.isabs(config.topsrcdir) + + path = os.path.normpath(path) + topsrcdir = os.path.normpath(config.topsrcdir) + + if path.startswith(topsrcdir): + return True + + external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split() + for external in external_dirs: + if not os.path.isabs(external): + external = os.path.join(config.topsrcdir, external) + external = os.path.normpath(external) + + if path.startswith(external): + return True + + return False + + class SandboxCalledError(SandboxError): """Represents an error resulting from calling the error() function.""" def __init__(self, file_stack, message): SandboxError.__init__(self, file_stack) self.message = message @@ -82,29 +114,48 @@ class MozbuildSandbox(Sandbox): to compute encountered relative paths. """ Sandbox.__init__(self, allowed_variables=VARIABLES) self.config = config topobjdir = os.path.abspath(config.topobjdir) - # This may not always hold true. If we ever have autogenerated mozbuild - # files in topobjdir, we'll need to change this. - assert os.path.normpath(path).startswith(os.path.normpath(config.topsrcdir)) - assert not os.path.normpath(path).startswith(os.path.normpath(topobjdir)) + topsrcdir = config.topsrcdir + if not path.startswith(topsrcdir): + external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split() + for external in external_dirs: + if not os.path.isabs(external): + external = os.path.join(config.topsrcdir, external) + + external = os.path.normpath(external) + + if not path.startswith(external): + continue - relpath = os.path.relpath(path, config.topsrcdir).replace(os.sep, '/') + topsrcdir = external + + # This is really hacky and should be replaced with something + # more robust. We assume that if an external source directory + # is in play that the main build system is built in a + # subdirectory of its topobjdir. Therefore, the topobjdir of + # the external source directory is the parent of our topobjdir. + topobjdir = os.path.dirname(topobjdir) + break + + self.topsrcdir = topsrcdir + + relpath = os.path.relpath(path, topsrcdir).replace(os.sep, '/') reldir = os.path.dirname(relpath) with self._globals.allow_all_writes() as d: - d['TOPSRCDIR'] = config.topsrcdir + d['TOPSRCDIR'] = topsrcdir d['TOPOBJDIR'] = topobjdir d['RELATIVEDIR'] = reldir - d['SRCDIR'] = os.path.join(config.topsrcdir, reldir).replace(os.sep, '/').rstrip('/') + d['SRCDIR'] = os.path.join(topsrcdir, reldir).replace(os.sep, '/').rstrip('/') d['OBJDIR'] = os.path.join(topobjdir, reldir).replace(os.sep, '/').rstrip('/') # config.status does not yet use unicode. However, mozbuild expects # unicode everywhere. So, decode binary into unicode as necessary. # Bug 844509 tracks a better way to do this. substs = {} for k, v in config.substs.items(): if not isinstance(v, text_type): @@ -113,50 +164,48 @@ class MozbuildSandbox(Sandbox): substs[k] = v d['CONFIG'] = ReadOnlyDefaultDict(substs, global_default=None) # Register functions. for name, func in FUNCTIONS.items(): d[name] = getattr(self, func[0]) - self._normalized_topsrcdir = os.path.normpath(config.topsrcdir) - def exec_file(self, path, filesystem_absolute=False): """Override exec_file to normalize paths and restrict file loading. If the path is absolute, behavior is governed by filesystem_absolute. If filesystem_absolute is True, the path is interpreted as absolute on the actual filesystem. If it is false, the path is treated as absolute within the current topsrcdir. If the path is not absolute, it will be treated as relative to the currently executing file. If there is no currently executing file, it will be treated as relative to topsrcdir. Paths will be rejected if they do not fall under topsrcdir. """ if os.path.isabs(path): if not filesystem_absolute: - path = os.path.normpath(os.path.join(self.config.topsrcdir, + path = os.path.normpath(os.path.join(self.topsrcdir, path[1:])) else: if len(self._execution_stack): path = os.path.normpath(os.path.join( os.path.dirname(self._execution_stack[-1]), path)) else: path = os.path.normpath(os.path.join( - self.config.topsrcdir, path)) + self.topsrcdir, path)) # realpath() is needed for true security. But, this isn't for security # protection, so it is omitted. normalized_path = os.path.normpath(path) - if not normalized_path.startswith(self._normalized_topsrcdir): + if not is_read_allowed(normalized_path, self.config): raise SandboxLoadError(list(self._execution_stack), sys.exc_info()[2], illegal_path=path) Sandbox.exec_file(self, path) def _add_tier_directory(self, tier, reldir, static=False): """Register a tier directory with the build.""" if isinstance(reldir, text_type): @@ -479,17 +528,16 @@ class BuildReader(object): """ def __init__(self, config): self.config = config self.topsrcdir = config.topsrcdir self._log = logging.getLogger(__name__) self._read_files = set() - self._normalized_topsrcdir = os.path.normpath(config.topsrcdir) self._execution_stack = [] def read_topsrcdir(self): """Read the tree of mozconfig files into a data structure. This starts with the tree's top-most mozbuild file and descends into all linked mozbuild files until all relevant files have been evaluated. @@ -610,19 +658,19 @@ class BuildReader(object): for relpath in dirs: child_path = os.path.join(curdir, relpath, 'moz.build') # Ensure we don't break out of the topsrcdir. We don't do realpath # because it isn't necessary. If there are symlinks in the srcdir, # that's not our problem. We're not a hosted application: we don't # need to worry about security too much. child_path = os.path.normpath(child_path) - if not child_path.startswith(self._normalized_topsrcdir): + if not is_read_allowed(child_path, self.config): raise SandboxValidationError( - 'Attempting to process file outside of topsrcdir: %s' % + 'Attempting to process file outside of allowed paths: %s' % child_path) if not descend: continue for res in self.read_mozbuild(child_path, read_tiers=False, filesystem_absolute=True): yield res
--- a/toolkit/toolkit.mozbuild +++ b/toolkit/toolkit.mozbuild @@ -204,16 +204,17 @@ add_tier_dir('platform', 'js/ductwork/de add_tier_dir('platform', 'other-licenses/snappy') if CONFIG['MOZ_GIO_COMPONENT']: add_tier_dir('platform', 'extensions/gio') # Applications can cheat and ask for code to be # built before libxul so it can be linked into libxul. add_tier_dir('platform', app_libxul_dirs) +add_tier_dir('platform', app_libxul_static_dirs, static=True) add_tier_dir('platform', 'toolkit/library') add_tier_dir('platform', 'xpcom/stub') if CONFIG['MOZ_REPLACE_MALLOC']: add_tier_dir('platform', 'memory/replace') if CONFIG['NS_TRACE_MALLOC']:
--- a/xulrunner/app.mozbuild +++ b/xulrunner/app.mozbuild @@ -1,14 +1,15 @@ # vim: set filetype=python: # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. app_libxul_dirs = [] +app_libxul_static_dirs = [] include('/toolkit/toolkit.mozbuild') if CONFIG['MOZ_EXTENSIONS']: add_tier_dir('app', 'extensions') if CONFIG['OS_ARCH'] == 'WINNT' and (CONFIG['ENABLE_TESTS'] or CONFIG['MOZILLA_OFFICIAL']): add_tier_dir('app', 'embedding/tests/winEmbed')