Bug 1132771 - Support and test for reading without a config object; r=glandium
☠☠ backed out by ce21e7a57cf8 ☠ ☠
authorGregory Szorc <gps@mozilla.com>
Thu, 26 Feb 2015 10:21:52 -0800
changeset 261579 c3a0cb4b45b17efa64bd8419b08d23d59ffc2883
parent 261578 91d34d3107faa777264cdcc9c89456bf4c289466
child 261580 7eed09d39b9fae0d3a08485129aebaa3b71ff04a
push id830
push userraliiev@mozilla.com
push dateFri, 19 Jun 2015 19:24:37 +0000
treeherdermozilla-release@932614382a68 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1132771
milestone39.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1132771 - Support and test for reading without a config object; r=glandium We want the ability to read data from any moz.build file without needing a full build configuration (running configure). This will enable tools to consume metadata by merely having a copy of the source code and nothing more. This commit creates the EmptyConfig object. It is a config object that - as its name implies - is empty. It will be used for reading moz.build files in "no config" mode. Many moz.build files make assumptions that variables in CONFIG are defined and that they are strings. We create the EmptyValue type that behaves like an empty unicode string. Since moz.build files also do some type checking, we carve an exemption for EmptyValue, just like we do for None. We add a test to verify that reading moz.build files in "no config" mode works. This required some minor changes to existing moz.build files to make them work in the new execution mode.
build/gyp.mozbuild
config/tests/test_mozbuild_reading.py
ipc/glue/moz.build
moz.build
netwerk/build/moz.build
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/util.py
--- a/build/gyp.mozbuild
+++ b/build/gyp.mozbuild
@@ -90,17 +90,17 @@ flavors = {
     'Darwin': 'mac' if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'cocoa' else 'ios',
     'SunOS': 'solaris',
     'GNU/kFreeBSD': 'freebsd',
     'DragonFly': 'dragonfly',
     'FreeBSD': 'freebsd',
     'NetBSD': 'netbsd',
     'OpenBSD': 'openbsd',
 }
-gyp_vars['OS'] = flavors[os]
+gyp_vars['OS'] = flavors.get(os)
 
 arches = {
     'x86_64': 'x64',
     'x86': 'ia32',
 }
 
 gyp_vars['target_arch'] = arches.get(CONFIG['CPU_ARCH'], CONFIG['CPU_ARCH'])
 
--- a/config/tests/test_mozbuild_reading.py
+++ b/config/tests/test_mozbuild_reading.py
@@ -5,40 +5,65 @@
 from __future__ import unicode_literals
 
 import os
 import unittest
 
 from mozunit import main
 
 from mozbuild.base import MozbuildObject
-from mozbuild.frontend.reader import BuildReader
+from mozbuild.frontend.reader import (
+    BuildReader,
+    EmptyConfig,
+)
 
 
 class TestMozbuildReading(unittest.TestCase):
     # This hack is needed to appease running in automation.
     def setUp(self):
         self._old_env = dict(os.environ)
         os.environ.pop('MOZCONFIG', None)
         os.environ.pop('MOZ_OBJDIR', None)
 
     def tearDown(self):
         os.environ.clear()
         os.environ.update(self._old_env)
 
+    def _mozbuilds(self, reader):
+        if not hasattr(self, '_mozbuild_paths'):
+            self._mozbuild_paths = set(reader.all_mozbuild_paths())
+
+        return self._mozbuild_paths
+
     def test_filesystem_traversal_reading(self):
         """Reading moz.build according to filesystem traversal works.
 
         We attempt to read every known moz.build file via filesystem traversal.
 
         If this test fails, it means that metadata extraction will fail.
         """
         mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
         config = mb.config_environment
         reader = BuildReader(config)
-        all_paths = set(reader.all_mozbuild_paths())
+        all_paths = self._mozbuilds(reader)
         paths, contexts = reader.read_relevant_mozbuilds(all_paths)
         self.assertEqual(set(paths), all_paths)
         self.assertGreaterEqual(len(contexts), len(paths))
 
+    def test_filesystem_traversal_no_config(self):
+        """Reading moz.build files via filesystem traversal mode with no build config.
+
+        This is similar to the above test except no build config is applied.
+        This will likely fail in more scenarios than the above test because a
+        lot of moz.build files assumes certain variables are present.
+        """
+        here = os.path.abspath(os.path.dirname(__file__))
+        root = os.path.normpath(os.path.join(here, '..', '..'))
+        config = EmptyConfig(root)
+        reader = BuildReader(config)
+        all_paths = self._mozbuilds(reader)
+        paths, contexts = reader.read_relevant_mozbuilds(all_paths)
+        self.assertEqual(set(paths.keys()), all_paths)
+        self.assertGreaterEqual(len(contexts), len(paths))
+
 
 if __name__ == '__main__':
     main()
--- a/ipc/glue/moz.build
+++ b/ipc/glue/moz.build
@@ -82,17 +82,17 @@ else:
 if CONFIG['OS_ARCH'] == 'Linux':
     UNIFIED_SOURCES += [
         'ProcessUtils_linux.cpp',
     ]
 elif CONFIG['OS_ARCH'] in ('DragonFly', 'FreeBSD', 'NetBSD', 'OpenBSD'):
     UNIFIED_SOURCES += [
         'ProcessUtils_bsd.cpp'
     ]
-elif CONFIG['OS_ARCH'] in ('Darwin'):
+elif CONFIG['OS_ARCH'] == 'Darwin':
     UNIFIED_SOURCES += [
         'ProcessUtils_mac.mm'
     ]
 else:
     UNIFIED_SOURCES += [
         'ProcessUtils_none.cpp',
     ]
 
--- a/moz.build
+++ b/moz.build
@@ -61,13 +61,13 @@ if CONFIG['COMPILE_ENVIRONMENT'] and not
         ]
 
     if CONFIG['BUILD_CTYPES']:
         DIRS += ['config/external/ffi']
     if CONFIG['USE_ICU']:
         DIRS += ['config/external/icu']
     DIRS += ['js/src']
 
-if not CONFIG['JS_STANDALONE']:
+if not CONFIG['JS_STANDALONE'] and CONFIG['MOZ_BUILD_APP']:
     # Bring in the configuration for the configured application.
     include('/' + CONFIG['MOZ_BUILD_APP'] + '/app.mozbuild')
 
 include('build/templates.mozbuild')
--- a/netwerk/build/moz.build
+++ b/netwerk/build/moz.build
@@ -28,17 +28,18 @@ LOCAL_INCLUDES += [
     '/netwerk/protocol/about',
     '/netwerk/protocol/app',
     '/netwerk/socket',
     '/netwerk/streamconv',
     '/netwerk/streamconv/converters',
 ]
 
 protocols = CONFIG['NECKO_PROTOCOLS'].copy()
-protocols.remove("about")
+if 'about' in protocols:
+    protocols.remove('about')
 LOCAL_INCLUDES += sorted([
     '/netwerk/protocol/%s' % d for d in protocols
 ])
 
 if CONFIG['OS_ARCH'] == 'WINNT':
     LOCAL_INCLUDES += [
         '/netwerk/system/win32',
     ]
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -31,16 +31,17 @@ import types
 
 from collections import (
     defaultdict,
     OrderedDict,
 )
 from io import StringIO
 
 from mozbuild.util import (
+    EmptyValue,
     memoize,
     ReadOnlyDefaultDict,
     ReadOnlyDict,
 )
 
 from mozbuild.backend.configenvironment import ConfigEnvironment
 
 from mozpack.files import FileFinder
@@ -77,16 +78,58 @@ else:
     text_type = str
     type_type = type
 
 
 def log(logger, level, action, params, formatter):
     logger.log(level, formatter, extra={'action': action, 'params': params})
 
 
+class EmptyConfig(object):
+    """A config object that is empty.
+
+    This config object is suitable for using with a BuildReader on a vanilla
+    checkout, without any existing configuration. The config is simply
+    bootstrapped from a top source directory path.
+    """
+    class PopulateOnGetDict(ReadOnlyDefaultDict):
+        """A variation on ReadOnlyDefaultDict that populates during .get().
+
+        This variation is needed because CONFIG uses .get() to access members.
+        Without it, None (instead of our EmptyValue types) would be returned.
+        """
+        def get(self, key, default=None):
+            return self[key]
+
+    def __init__(self, topsrcdir):
+        self.topsrcdir = topsrcdir
+        self.topobjdir = ''
+
+        self.substs = self.PopulateOnGetDict(EmptyValue, {
+            # These 2 variables are used semi-frequently and it isn't worth
+            # changing all the instances.
+            b'MOZ_APP_NAME': b'empty',
+            b'MOZ_CHILD_PROCESS_NAME': b'empty',
+            # Set manipulations are performed within the moz.build files. But
+            # set() is not an exposed symbol, so we can't create an empty set.
+            b'NECKO_PROTOCOLS': set(),
+            # Needed to prevent js/src's config.status from loading.
+            b'JS_STANDALONE': b'1',
+        })
+        udict = {}
+        for k, v in self.substs.items():
+            if isinstance(v, str):
+                udict[k.decode('utf-8')] = v.decode('utf-8')
+            else:
+                udict[k] = v
+        self.substs_unicode = self.PopulateOnGetDict(EmptyValue, udict)
+        self.defines = self.substs
+        self.external_source_dir = None
+
+
 def is_read_allowed(path, config):
     """Whether we are allowed to load a mozbuild file at the specified path.
 
     This is used as cheap security to ensure the build is isolated to known
     source directories.
 
     We are allowed to read from the main source directory and any defined
     external source directories. The latter is to allow 3rd party applications
--- a/python/mozbuild/mozbuild/util.py
+++ b/python/mozbuild/mozbuild/util.py
@@ -13,16 +13,17 @@ import difflib
 import errno
 import functools
 import hashlib
 import itertools
 import os
 import stat
 import sys
 import time
+import types
 
 from collections import (
     defaultdict,
     OrderedDict,
 )
 from StringIO import StringIO
 
 
@@ -45,16 +46,27 @@ def hash_file(path, hasher=None):
             if not len(data):
                 break
 
             h.update(data)
 
     return h.hexdigest()
 
 
+class EmptyValue(unicode):
+    """A dummy type that behaves like an empty string and sequence.
+
+    This type exists in order to support
+    :py:class:`mozbuild.frontend.reader.EmptyConfig`. It should likely not be
+    used elsewhere.
+    """
+    def __init__(self):
+        super(EmptyValue, self).__init__()
+
+
 class ReadOnlyDict(dict):
     """A read-only dictionary."""
     def __init__(self, *args, **kwargs):
         dict.__init__(self, *args, **kwargs)
 
     def __delitem__(self, key):
         raise Exception('Object does not support deletion.')
 
@@ -249,28 +261,28 @@ class ListMixin(object):
 
     def __setslice__(self, i, j, sequence):
         if not isinstance(sequence, list):
             raise ValueError('List can only be sliced with other list instances.')
 
         return super(ListMixin, self).__setslice__(i, j, sequence)
 
     def __add__(self, other):
-        # Allow None is a special case because it makes undefined variable
-        # references in moz.build behave better.
-        other = [] if other is None else other
+        # Allow None and EmptyValue is a special case because it makes undefined
+        # variable references in moz.build behave better.
+        other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other
         if not isinstance(other, list):
             raise ValueError('Only lists can be appended to lists.')
 
         new_list = self.__class__(self)
         new_list.extend(other)
         return new_list
 
     def __iadd__(self, other):
-        other = [] if other is None else other
+        other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other
         if not isinstance(other, list):
             raise ValueError('Only lists can be appended to lists.')
 
         return super(ListMixin, self).__iadd__(other)
 
 
 class List(ListMixin, list):
     """A list specialized for moz.build environments.