Bug 1035599: Merge the mozilla-central and comm-central objdir, r=gps
authorJoshua Cranmer <Pidgeot18@gmail.com>
Thu, 07 Aug 2014 19:27:58 -0500
changeset 220086 e234b61f711e29228341dc343d1d269dd9942c27
parent 220085 497e1d4921a0bedd6aa27877d2d23f63ba59971a
child 220087 698584a65cdbbfcde8cd720b176e771ff3154570
push id3979
push userraliiev@mozilla.com
push dateMon, 13 Oct 2014 16:35:44 +0000
treeherdermozilla-beta@30f2cc610691 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1035599
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1035599: Merge the mozilla-central and comm-central objdir, r=gps This change merges mozilla-central and comm-central into having a single topobjdir file but retaining two topsrcdirs (and two build systems). This state is hopefully only the first part of a series of changes that eliminate the comm-central build system partial clone completely.
Makefile.in
configure.in
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/frontend/test_sandbox.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -323,17 +323,18 @@ js/src/export config/host: build/clang-p
 endif
 
 # Interdependencies that moz.build world don't know about yet for compilation.
 # Note some others are hardcoded or "guessed" in recursivemake.py and emitter.py
 ifeq ($(MOZ_WIDGET_TOOLKIT),gtk3)
 toolkit/library/target: widget/gtk/mozgtk/gtk3/target
 endif
 ifdef MOZ_LDAP_XPCOM
-toolkit/library/target: ../ldap/target
+ldap/target: config/external/nss/target mozglue/build/target
+toolkit/library/target: ldap/target
 endif
 ifndef MOZ_FOLD_LIBS
 ifndef MOZ_NATIVE_SQLITE
 config/external/nss/target: db/sqlite3/src/target
 endif
 endif
 ifeq ($(MOZ_REPLACE_MALLOC_LINKAGE),dummy library)
 mozglue/build/target: memory/replace/dummy/target
--- a/configure.in
+++ b/configure.in
@@ -2519,20 +2519,20 @@ if test "$GNU_CC" -a "$OS_TARGET" != WIN
   AC_DEFINE(HAVE_VISIBILITY_ATTRIBUTE)
   case "$OS_TARGET" in
   Darwin)
     VISIBILITY_FLAGS='-fvisibility=hidden'
     ;;
   *)
     case $GCC_VERSION in
     4.4*|4.6*)
-      VISIBILITY_FLAGS='-I$(DIST)/system_wrappers -include $(topsrcdir)/config/gcc_hidden_dso_handle.h'
+      VISIBILITY_FLAGS='-I$(DIST)/system_wrappers -include $(MOZILLA_DIR)/config/gcc_hidden_dso_handle.h'
       ;;
     *)
-      VISIBILITY_FLAGS='-I$(DIST)/system_wrappers -include $(topsrcdir)/config/gcc_hidden.h'
+      VISIBILITY_FLAGS='-I$(DIST)/system_wrappers -include $(MOZILLA_DIR)/config/gcc_hidden.h'
       ;;
     esac
     WRAP_SYSTEM_INCLUDES=1
     ;;
   esac
 fi         # GNU_CC
 
 # visibility hidden flag for Sun Studio on Solaris
@@ -9029,20 +9029,23 @@ if test -z "$direct_nspr_config"; then
     RC=
 fi
 
 unset MAKEFILES
 unset CONFIG_FILES
 
 # Run all configure scripts specified by a subconfigure
 if test -n "$_subconfigure_subdir"; then
+  _save_srcdir="$srcdir"
+  srcdir="$srcdir/.."
   _save_ac_configure_args="$ac_configure_args"
   ac_configure_args="$_subconfigure_config_args"
   AC_OUTPUT_SUBDIRS("$_subconfigure_subdir")
   ac_configure_args="$_save_ac_configure_args"
+  srcdir="$_save_srcdir"
 fi
 
 # No need to run subconfigures when building with LIBXUL_SDK_DIR
 if test "$COMPILE_ENVIRONMENT" -a -z "$LIBXUL_SDK_DIR"; then
 
 export WRAP_LDFLAGS
 
 if test -n "$_WRAP_MALLOC"; then
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -84,17 +84,18 @@ class BackendMakeFile(object):
     invalidate all make targets across the whole tree! This would effectively
     undermine incremental builds as any mozbuild change would cause the entire
     tree to rebuild!
 
     The solution is to not update the mtimes of backend.mk files unless they
     actually change. We use FileAvoidWrite to accomplish this.
     """
 
-    def __init__(self, srcdir, objdir, environment, topobjdir):
+    def __init__(self, srcdir, objdir, environment, topsrcdir, topobjdir):
+        self.topsrcdir = topsrcdir
         self.srcdir = srcdir
         self.objdir = objdir
         self.relobjdir = mozpath.relpath(objdir, topobjdir)
         self.environment = environment
         self.name = mozpath.join(objdir, 'backend.mk')
 
         # XPIDLFiles attached to this file.
         self.idls = []
@@ -324,17 +325,17 @@ class RecursiveMakeBackend(CommonBackend
         """Write out build files necessary to build with recursive make."""
 
         if not isinstance(obj, SandboxDerived):
             return
 
         if obj.objdir not in self._backend_files:
             self._backend_files[obj.objdir] = \
                 BackendMakeFile(obj.srcdir, obj.objdir, obj.config,
-                    self.environment.topobjdir)
+                    obj.topsrcdir, self.environment.topobjdir)
         backend_file = self._backend_files[obj.objdir]
 
         CommonBackend.consume_object(self, obj)
 
         # CommonBackend handles XPIDLFile and TestManifest, but we want to do
         # some extra things for them.
         if isinstance(obj, XPIDLFile):
             backend_file.idls.append(obj)
@@ -691,17 +692,17 @@ class RecursiveMakeBackend(CommonBackend
                             skiplist.remove(bf.relobjdir)
                 else:
                     self.log(logging.DEBUG, 'stub_makefile',
                         {'path': makefile}, 'Creating stub Makefile: {path}')
 
                 obj = self.Substitution()
                 obj.output_path = makefile
                 obj.input_path = makefile_in
-                obj.topsrcdir = bf.environment.topsrcdir
+                obj.topsrcdir = backend_file.topsrcdir
                 obj.topobjdir = bf.environment.topobjdir
                 obj.config = bf.environment
                 self._create_makefile(obj, stub=stub)
                 with open(obj.output_path) as fh:
                     content = fh.read()
                     # Skip every directory but those with a Makefile
                     # containing a tools target, or XPI_PKGNAME or
                     # INSTALL_EXTENSION_ID.
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -131,44 +131,35 @@ class MozbuildSandbox(Sandbox):
 
         self.config = config
         self.metadata = dict(metadata)
 
         topobjdir = mozpath.abspath(config.topobjdir)
         topsrcdir = config.topsrcdir
         norm_topsrcdir = mozpath.normpath(topsrcdir)
 
-        if not path.startswith(norm_topsrcdir):
-            external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split()
-            for external in external_dirs:
-                external = mozpath.normpath(external)
+        self.external_source_dirs = []
+        external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split()
+        for external in external_dirs:
+            external = mozpath.normpath(external)
 
-                if not os.path.isabs(external):
-                    external = mozpath.join(config.topsrcdir, external)
+            if not os.path.isabs(external):
+                external = mozpath.join(config.topsrcdir, external)
 
-                external = mozpath.normpath(external)
+            external = mozpath.normpath(external)
+            self.external_source_dirs.append(external)
 
+
+        if not path.startswith(norm_topsrcdir):
+            for external in self.external_source_dirs:
                 if not path.startswith(external):
                     continue
 
                 topsrcdir = external
 
-                # This is really hacky and should be replaced with something
-                # more robust. We assume that if an external source directory
-                # is in play that the main build system is built in a
-                # subdirectory of its topobjdir. Therefore, the topobjdir of
-                # the external source directory is the parent of our topobjdir.
-                topobjdir = mozpath.dirname(topobjdir)
-
-                # This is suboptimal because we load the config.status multiple
-                # times. We should consider caching it, possibly by moving this
-                # code up to the reader.
-                config = ConfigEnvironment.from_config_status(
-                    mozpath.join(topobjdir, 'config.status'))
-                self.config = config
                 break
 
         self.topsrcdir = topsrcdir
 
         relpath = mozpath.relpath(path, topsrcdir)
         reldir = mozpath.dirname(relpath)
 
         if mozpath.dirname(relpath) == 'js/src' and \
@@ -191,52 +182,66 @@ class MozbuildSandbox(Sandbox):
             # Register functions.
             for name, func in FUNCTIONS.items():
                 d[name] = getattr(self, func[0])
 
             # Initialize the exports that we need in the global.
             extra_vars = self.metadata.get('exports', dict())
             self._globals.update(extra_vars)
 
-    def exec_file(self, path, filesystem_absolute=False):
-        """Override exec_file to normalize paths and restrict file loading.
+    def normalize_path(self, path, filesystem_absolute=False, srcdir=None):
+        """Normalizes paths.
 
         If the path is absolute, behavior is governed by filesystem_absolute.
         If filesystem_absolute is True, the path is interpreted as absolute on
         the actual filesystem. If it is false, the path is treated as absolute
         within the current topsrcdir.
 
         If the path is not absolute, it will be treated as relative to the
         currently executing file. If there is no currently executing file, it
         will be treated as relative to topsrcdir.
-
-        Paths will be rejected if they do not fall under topsrcdir.
         """
         if os.path.isabs(path):
-            if not filesystem_absolute:
-                path = mozpath.normpath(mozpath.join(self.topsrcdir,
-                    path[1:]))
-
+            # If the path isn't in Unix-style, this is going to be problematic.
+            assert path[0] == '/'
+            if filesystem_absolute:
+                return path
+            for root in [self.topsrcdir] + self.external_source_dirs:
+                # mozpath.join would ignore the self.topsrcdir argument if we
+                # passed in the absolute path, so omit the leading /
+                p = mozpath.normpath(mozpath.join(root, path[1:]))
+                if os.path.exists(p):
+                    return p
+            # mozpath.join would ignore the self.topsrcdir argument if we passed
+            # in the absolute path, so omit the leading /
+            return mozpath.normpath(mozpath.join(self.topsrcdir, path[1:]))
+        elif srcdir:
+            return mozpath.normpath(mozpath.join(srcdir, path))
+        elif len(self._execution_stack):
+            return mozpath.normpath(mozpath.join(
+                mozpath.dirname(self._execution_stack[-1]), path))
         else:
-            if len(self._execution_stack):
-                path = mozpath.normpath(mozpath.join(
-                    mozpath.dirname(self._execution_stack[-1]),
-                    path))
-            else:
-                path = mozpath.normpath(mozpath.join(
-                    self.topsrcdir, path))
+            return mozpath.normpath(mozpath.join(self.topsrcdir, path))
+
+    def exec_file(self, path, filesystem_absolute=False):
+        """Override exec_file to normalize paths and restrict file loading.
+
+        Paths will be rejected if they do not fall under topsrcdir or one of
+        the external roots.
+        """
 
         # realpath() is needed for true security. But, this isn't for security
         # protection, so it is omitted.
-        normalized_path = mozpath.normpath(path)
+        normalized_path = self.normalize_path(path,
+            filesystem_absolute=filesystem_absolute)
         if not is_read_allowed(normalized_path, self.config):
             raise SandboxLoadError(list(self._execution_stack),
                 sys.exc_info()[2], illegal_path=path)
 
-        Sandbox.exec_file(self, path)
+        Sandbox.exec_file(self, normalized_path)
 
     def _add_java_jar(self, name):
         """Add a Java JAR build target."""
         if not name:
             raise Exception('Java JAR cannot be registered without a name')
 
         if '/' in name or '\\' in name or '.jar' in name:
             raise Exception('Java JAR names must not include slashes or'
@@ -869,26 +874,29 @@ class BuildReader(object):
                 # definition is external to the build system.
                 for d in values['regular']:
                     if d in recurse_info:
                         raise SandboxValidationError(
                             'Tier directory (%s) registered multiple '
                             'times in %s' % (d, tier), sandbox)
                     recurse_info[d] = {'tier': tier,
                                        'parent': sandbox['RELATIVEDIR'],
+                                       'check_external': True,
                                        'var': 'DIRS'}
 
         for relpath, child_metadata in recurse_info.items():
-            child_path = mozpath.join(curdir, relpath, 'moz.build')
+            if 'check_external' in child_metadata:
+                relpath = '/' + relpath
+            child_path = sandbox.normalize_path(mozpath.join(relpath,
+                'moz.build'), srcdir=curdir)
 
             # Ensure we don't break out of the topsrcdir. We don't do realpath
             # because it isn't necessary. If there are symlinks in the srcdir,
             # that's not our problem. We're not a hosted application: we don't
             # need to worry about security too much.
-            child_path = mozpath.normpath(child_path)
             if not is_read_allowed(child_path, sandbox.config):
                 raise SandboxValidationError(
                     'Attempting to process file outside of allowed paths: %s' %
                         child_path, sandbox)
 
             if not descend:
                 continue
 
--- a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py
@@ -222,18 +222,17 @@ add_tier_dir('t1', 'bat')
         self.assertEqual(len(sandbox.all_paths), 2)
 
     def test_include_outside_topsrcdir(self):
         sandbox = self.sandbox(data_path='include-outside-topsrcdir')
 
         with self.assertRaises(SandboxLoadError) as se:
             sandbox.exec_file('relative.build')
 
-        expected = mozpath.join(test_data_path, 'moz.build')
-        self.assertEqual(se.exception.illegal_path, expected)
+        self.assertEqual(se.exception.illegal_path, '../moz.build')
 
     def test_include_error_stack(self):
         # Ensure the path stack is reported properly in exceptions.
         sandbox = self.sandbox(data_path='include-file-stack')
 
         with self.assertRaises(SandboxExecutionError) as se:
             sandbox.exec_file('moz.build')