Bug 1132771 - Support and test for reading without a config object; r=glandium
authorGregory Szorc <gps@mozilla.com>
Thu, 26 Feb 2015 10:21:52 -0800
changeset 231230 5a1306a14a8cfbf0e5bc0b65e4db84156a434b30
parent 231229 a6b72bee55a8b7a2a0f7e4e6e8b8a3c5da7946b7
child 231231 c5a0d6e73532abd7966b84107d352c2fc2910c49
push id11549
push usergszorc@mozilla.com
push dateMon, 02 Mar 2015 17:47:28 +0000
treeherderfx-team@f19c915183c2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1132771
milestone39.0a1
Bug 1132771 - Support and test for reading without a config object; r=glandium We want the ability to read data from any moz.build file without needing a full build configuration (running configure). This will enable tools to consume metadata by merely having a copy of the source code and nothing more. This commit creates the EmptyConfig object. It is a config object that - as its name implies - is empty. It will be used for reading moz.build files in "no config" mode. Many moz.build files make assumptions that variables in CONFIG are defined and that they are strings. We create the EmptyValue type that behaves like an empty unicode string. Since moz.build files also do some type checking, we carve an exemption for EmptyValue, just like we do for None. We add a test to verify that reading moz.build files in "no config" mode works. This required some minor changes to existing moz.build files to make them work in the new execution mode.
build/gyp.mozbuild
config/tests/test_mozbuild_reading.py
ipc/glue/moz.build
moz.build
netwerk/build/moz.build
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/util.py
--- a/build/gyp.mozbuild
+++ b/build/gyp.mozbuild
@@ -90,17 +90,17 @@ flavors = {
     'Darwin': 'mac' if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'cocoa' else 'ios',
     'SunOS': 'solaris',
     'GNU/kFreeBSD': 'freebsd',
     'DragonFly': 'dragonfly',
     'FreeBSD': 'freebsd',
     'NetBSD': 'netbsd',
     'OpenBSD': 'openbsd',
 }
-gyp_vars['OS'] = flavors[os]
+gyp_vars['OS'] = flavors.get(os)
 
 arches = {
     'x86_64': 'x64',
     'x86': 'ia32',
 }
 
 gyp_vars['target_arch'] = arches.get(CONFIG['CPU_ARCH'], CONFIG['CPU_ARCH'])
 
--- a/config/tests/test_mozbuild_reading.py
+++ b/config/tests/test_mozbuild_reading.py
@@ -5,41 +5,66 @@
 from __future__ import unicode_literals
 
 import os
 import unittest
 
 from mozunit import main
 
 from mozbuild.base import MozbuildObject
-from mozbuild.frontend.reader import BuildReader
+from mozbuild.frontend.reader import (
+    BuildReader,
+    EmptyConfig,
+)
 
 
 class TestMozbuildReading(unittest.TestCase):
     # This hack is needed to appease running in automation.
     def setUp(self):
         self._old_env = dict(os.environ)
         os.environ.pop('MOZCONFIG', None)
         os.environ.pop('MOZ_OBJDIR', None)
 
     def tearDown(self):
         os.environ.clear()
         os.environ.update(self._old_env)
 
+    def _mozbuilds(self, reader):
+        if not hasattr(self, '_mozbuild_paths'):
+            self._mozbuild_paths = set(reader.all_mozbuild_paths())
+
+        return self._mozbuild_paths
+
     @unittest.skip('failing in SpiderMonkey builds')
     def test_filesystem_traversal_reading(self):
         """Reading moz.build according to filesystem traversal works.
 
         We attempt to read every known moz.build file via filesystem traversal.
 
         If this test fails, it means that metadata extraction will fail.
         """
         mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
         config = mb.config_environment
         reader = BuildReader(config)
-        all_paths = set(reader.all_mozbuild_paths())
+        all_paths = self._mozbuilds(reader)
         paths, contexts = reader.read_relevant_mozbuilds(all_paths)
         self.assertEqual(set(paths), all_paths)
         self.assertGreaterEqual(len(contexts), len(paths))
 
+    def test_filesystem_traversal_no_config(self):
+        """Reading moz.build files via filesystem traversal mode with no build config.
+
+        This is similar to the above test except no build config is applied.
+        This will likely fail in more scenarios than the above test because a
+        lot of moz.build files assumes certain variables are present.
+        """
+        here = os.path.abspath(os.path.dirname(__file__))
+        root = os.path.normpath(os.path.join(here, '..', '..'))
+        config = EmptyConfig(root)
+        reader = BuildReader(config)
+        all_paths = self._mozbuilds(reader)
+        paths, contexts = reader.read_relevant_mozbuilds(all_paths)
+        self.assertEqual(set(paths.keys()), all_paths)
+        self.assertGreaterEqual(len(contexts), len(paths))
+
 
 if __name__ == '__main__':
     main()
--- a/ipc/glue/moz.build
+++ b/ipc/glue/moz.build
@@ -82,17 +82,17 @@ else:
 if CONFIG['OS_ARCH'] == 'Linux':
     UNIFIED_SOURCES += [
         'ProcessUtils_linux.cpp',
     ]
 elif CONFIG['OS_ARCH'] in ('DragonFly', 'FreeBSD', 'NetBSD', 'OpenBSD'):
     UNIFIED_SOURCES += [
         'ProcessUtils_bsd.cpp'
     ]
-elif CONFIG['OS_ARCH'] in ('Darwin'):
+elif CONFIG['OS_ARCH'] == 'Darwin':
     UNIFIED_SOURCES += [
         'ProcessUtils_mac.mm'
     ]
 else:
     UNIFIED_SOURCES += [
         'ProcessUtils_none.cpp',
     ]
 
--- a/moz.build
+++ b/moz.build
@@ -61,13 +61,13 @@ if CONFIG['COMPILE_ENVIRONMENT'] and not
         ]
 
     if CONFIG['BUILD_CTYPES']:
         DIRS += ['config/external/ffi']
     if CONFIG['USE_ICU']:
         DIRS += ['config/external/icu']
     DIRS += ['js/src']
 
-if not CONFIG['JS_STANDALONE']:
+if not CONFIG['JS_STANDALONE'] and CONFIG['MOZ_BUILD_APP']:
     # Bring in the configuration for the configured application.
     include('/' + CONFIG['MOZ_BUILD_APP'] + '/app.mozbuild')
 
 include('build/templates.mozbuild')
--- a/netwerk/build/moz.build
+++ b/netwerk/build/moz.build
@@ -28,17 +28,18 @@ LOCAL_INCLUDES += [
     '/netwerk/protocol/about',
     '/netwerk/protocol/app',
     '/netwerk/socket',
     '/netwerk/streamconv',
     '/netwerk/streamconv/converters',
 ]
 
 protocols = CONFIG['NECKO_PROTOCOLS'].copy()
-protocols.remove("about")
+if 'about' in protocols:
+    protocols.remove('about')
 LOCAL_INCLUDES += sorted([
     '/netwerk/protocol/%s' % d for d in protocols
 ])
 
 if CONFIG['OS_ARCH'] == 'WINNT':
     LOCAL_INCLUDES += [
         '/netwerk/system/win32',
     ]
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -31,16 +31,17 @@ import types
 
 from collections import (
     defaultdict,
     OrderedDict,
 )
 from io import StringIO
 
 from mozbuild.util import (
+    EmptyValue,
     memoize,
     ReadOnlyDefaultDict,
     ReadOnlyDict,
 )
 
 from mozbuild.backend.configenvironment import ConfigEnvironment
 
 from mozpack.files import FileFinder
@@ -77,16 +78,58 @@ else:
     text_type = str
     type_type = type
 
 
 def log(logger, level, action, params, formatter):
     logger.log(level, formatter, extra={'action': action, 'params': params})
 
 
+class EmptyConfig(object):
+    """A config object that is empty.
+
+    This config object is suitable for using with a BuildReader on a vanilla
+    checkout, without any existing configuration. The config is simply
+    bootstrapped from a top source directory path.
+    """
+    class PopulateOnGetDict(ReadOnlyDefaultDict):
+        """A variation on ReadOnlyDefaultDict that populates during .get().
+
+        This variation is needed because CONFIG uses .get() to access members.
+        Without it, None (instead of our EmptyValue types) would be returned.
+        """
+        def get(self, key, default=None):
+            return self[key]
+
+    def __init__(self, topsrcdir):
+        self.topsrcdir = topsrcdir
+        self.topobjdir = ''
+
+        self.substs = self.PopulateOnGetDict(EmptyValue, {
+            # These 2 variables are used semi-frequently and it isn't worth
+            # changing all the instances.
+            b'MOZ_APP_NAME': b'empty',
+            b'MOZ_CHILD_PROCESS_NAME': b'empty',
+            # Set manipulations are performed within the moz.build files. But
+            # set() is not an exposed symbol, so we can't create an empty set.
+            b'NECKO_PROTOCOLS': set(),
+            # Needed to prevent js/src's config.status from loading.
+            b'JS_STANDALONE': b'1',
+        })
+        udict = {}
+        for k, v in self.substs.items():
+            if isinstance(v, str):
+                udict[k.decode('utf-8')] = v.decode('utf-8')
+            else:
+                udict[k] = v
+        self.substs_unicode = self.PopulateOnGetDict(EmptyValue, udict)
+        self.defines = self.substs
+        self.external_source_dir = None
+
+
 def is_read_allowed(path, config):
     """Whether we are allowed to load a mozbuild file at the specified path.
 
     This is used as cheap security to ensure the build is isolated to known
     source directories.
 
     We are allowed to read from the main source directory and any defined
     external source directories. The latter is to allow 3rd party applications
--- a/python/mozbuild/mozbuild/util.py
+++ b/python/mozbuild/mozbuild/util.py
@@ -13,16 +13,17 @@ import difflib
 import errno
 import functools
 import hashlib
 import itertools
 import os
 import stat
 import sys
 import time
+import types
 
 from collections import (
     defaultdict,
     OrderedDict,
 )
 from StringIO import StringIO
 
 
@@ -45,16 +46,27 @@ def hash_file(path, hasher=None):
             if not len(data):
                 break
 
             h.update(data)
 
     return h.hexdigest()
 
 
+class EmptyValue(unicode):
+    """A dummy type that behaves like an empty string and sequence.
+
+    This type exists in order to support
+    :py:class:`mozbuild.frontend.reader.EmptyConfig`. It should likely not be
+    used elsewhere.
+    """
+    def __init__(self):
+        super(EmptyValue, self).__init__()
+
+
 class ReadOnlyDict(dict):
     """A read-only dictionary."""
     def __init__(self, *args, **kwargs):
         dict.__init__(self, *args, **kwargs)
 
     def __delitem__(self, key):
         raise Exception('Object does not support deletion.')
 
@@ -249,28 +261,28 @@ class ListMixin(object):
 
     def __setslice__(self, i, j, sequence):
         if not isinstance(sequence, list):
             raise ValueError('List can only be sliced with other list instances.')
 
         return super(ListMixin, self).__setslice__(i, j, sequence)
 
     def __add__(self, other):
-        # Allow None is a special case because it makes undefined variable
-        # references in moz.build behave better.
-        other = [] if other is None else other
+        # Allow None and EmptyValue is a special case because it makes undefined
+        # variable references in moz.build behave better.
+        other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other
         if not isinstance(other, list):
             raise ValueError('Only lists can be appended to lists.')
 
         new_list = self.__class__(self)
         new_list.extend(other)
         return new_list
 
     def __iadd__(self, other):
-        other = [] if other is None else other
+        other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other
         if not isinstance(other, list):
             raise ValueError('Only lists can be appended to lists.')
 
         return super(ListMixin, self).__iadd__(other)
 
 
 class List(ListMixin, list):
     """A list specialized for moz.build environments.