Bug 1270446 - Move CompilerPreprocessor, FakeCompiler and their tests to a separate file. r=chmanchester
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 22 Apr 2016 10:03:56 +0900
changeset 296419 8e38b67846f12fc5423810c67a1f25721f57cdc4
parent 296418 6faf0619a211d3841089aabb2405183483eb4a9a
child 296420 471bf0db284398af3ac4c83615e56ad22a0e2fe8
push id19113
push userkwierso@gmail.com
push dateFri, 06 May 2016 22:59:05 +0000
treeherderfx-team@ba635ae2a090 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1270446
milestone49.0a1
Bug 1270446 - Move CompilerPreprocessor, FakeCompiler and their tests to a separate file. r=chmanchester This makes the toolchain.configure tests more prominent in the file.
python/moz.build
python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
--- a/python/moz.build
+++ b/python/moz.build
@@ -37,16 +37,17 @@ PYTHON_UNIT_TESTS += [
     'mozbuild/mozbuild/test/backend/test_recursivemake.py',
     'mozbuild/mozbuild/test/backend/test_visualstudio.py',
     'mozbuild/mozbuild/test/compilation/test_warnings.py',
     'mozbuild/mozbuild/test/configure/test_checks_configure.py',
     'mozbuild/mozbuild/test/configure/test_configure.py',
     'mozbuild/mozbuild/test/configure/test_moz_configure.py',
     'mozbuild/mozbuild/test/configure/test_options.py',
     'mozbuild/mozbuild/test/configure/test_toolchain_configure.py',
+    'mozbuild/mozbuild/test/configure/test_toolchain_helpers.py',
     'mozbuild/mozbuild/test/configure/test_util.py',
     'mozbuild/mozbuild/test/controller/test_ccachestats.py',
     'mozbuild/mozbuild/test/controller/test_clobber.py',
     'mozbuild/mozbuild/test/frontend/test_context.py',
     'mozbuild/mozbuild/test/frontend/test_emitter.py',
     'mozbuild/mozbuild/test/frontend/test_namespaces.py',
     'mozbuild/mozbuild/test/frontend/test_reader.py',
     'mozbuild/mozbuild/test/frontend/test_sandbox.py',
--- a/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
@@ -1,242 +1,24 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
 import os
-import re
-import types
-import unittest
 
-from collections import defaultdict
-from fnmatch import fnmatch
 from StringIO import StringIO
-from textwrap import dedent
 
-from mozunit import (
-    main,
-    MockedOpen,
-)
+from mozunit import main
 
 from common import BaseConfigureTest
-from mozbuild.preprocessor import Preprocessor
 from mozpack import path as mozpath
-
-
-class CompilerPreprocessor(Preprocessor):
-    VARSUBST = re.compile('(?P<VAR>\w+)', re.U)
-    NON_WHITESPACE = re.compile('\S')
-
-    def __init__(self, *args, **kwargs):
-        Preprocessor.__init__(self, *args, **kwargs)
-        self.do_filter('c_substitution')
-        self.setMarker('#\s*')
-
-    def do_if(self, *args, **kwargs):
-        # The C preprocessor handles numbers following C rules, which is a
-        # different handling than what our Preprocessor does out of the box.
-        # Hack around it enough that the configure tests work properly.
-        context = self.context
-        def normalize_numbers(value):
-            if isinstance(value, types.StringTypes):
-                if value[-1:] == 'L' and value[:-1].isdigit():
-                    value = int(value[:-1])
-            return value
-        self.context = self.Context(
-            (k, normalize_numbers(v)) for k, v in context.iteritems()
-        )
-        try:
-            return Preprocessor.do_if(self, *args, **kwargs)
-        finally:
-            self.context = context
-
-    class Context(dict):
-        def __missing__(self, key):
-            return None
-
-    def filter_c_substitution(self, line):
-        def repl(matchobj):
-            varname = matchobj.group('VAR')
-            if varname in self.context:
-                result = str(self.context[varname])
-                # The C preprocessor inserts whitespaces around expanded
-                # symbols.
-                start, end = matchobj.span('VAR')
-                if self.NON_WHITESPACE.match(line[start-1:start]):
-                    result = ' ' + result
-                if self.NON_WHITESPACE.match(line[end:end+1]):
-                    result = result + ' '
-                return result
-            return matchobj.group(0)
-        return self.VARSUBST.sub(repl, line)
-
-
-class TestCompilerPreprocessor(unittest.TestCase):
-    def test_expansion(self):
-        pp = CompilerPreprocessor({
-            'A': 1,
-            'B': '2',
-            'C': 'c',
-        })
-        pp.out = StringIO()
-        input = StringIO('A.B.C')
-        input.name = 'foo'
-        pp.do_include(input)
-
-        self.assertEquals(pp.out.getvalue(), '1 . 2 . c')
-
-    def test_condition(self):
-        pp = CompilerPreprocessor({
-            'A': 1,
-            'B': '2',
-            'C': '0L',
-        })
-        pp.out = StringIO()
-        input = StringIO(dedent('''\
-            #ifdef A
-            IFDEF_A
-            #endif
-            #if A
-            IF_A
-            #endif
-            #  if B
-            IF_B
-            #  else
-            IF_NOT_B
-            #  endif
-            #if !C
-            IF_NOT_C
-            #else
-            IF_C
-            #endif
-        '''))
-        input.name = 'foo'
-        pp.do_include(input)
-
-        self.assertEquals('IFDEF_A\nIF_A\nIF_B\nIF_NOT_C\n', pp.out.getvalue())
-
-
-class FakeCompiler(object):
-    '''Defines a fake compiler for use in toolchain tests below.
-
-    The definition given when creating an instance can have one of two
-    forms:
-    - a dict giving preprocessor symbols and their respective value, e.g.
-        { '__GNUC__': 4, '__STDC__': 1 }
-    - a dict associating flags to preprocessor symbols. An entry for `None`
-      is required in this case. Those are the baseline preprocessor symbols.
-      Additional entries describe additional flags to set or existing flags
-      to unset (with a value of `False`).
-        {
-          None: { '__GNUC__': 4, '__STDC__': 1, '__STRICT_ANSI__': 1 },
-          '-std=gnu99': { '__STDC_VERSION__': '199901L',
-                          '__STRICT_ANSI__': False },
-        }
-      With the dict above, invoking the preprocessor with no additional flags
-      would define __GNUC__, __STDC__ and __STRICT_ANSI__, and with -std=gnu99,
-      __GNUC__, __STDC__, and __STDC_VERSION__ (__STRICT_ANSI__ would be
-      unset).
-      It is also possible to have different symbols depending on the source
-      file extension. In this case, the key is '*.ext'. e.g.
-        {
-          '*.c': { '__STDC__': 1 },
-          '*.cpp': { '__cplusplus': '199711L' },
-        }
-    '''
-    def __init__(self, definition):
-        if definition.get(None) is None:
-            definition = {None: definition}
-        self._definition = definition
-
-    def __call__(self, stdin, args):
-        files = [arg for arg in args if not arg.startswith('-')]
-        flags = [arg for arg in args if arg.startswith('-')]
-        if '-E' in flags:
-            assert len(files) == 1
-            file = files[0]
-            pp = CompilerPreprocessor(self._definition[None])
-
-            def apply_defn(defn):
-                for k, v in defn.iteritems():
-                    if v is False:
-                        if k in pp.context:
-                            del pp.context[k]
-                    else:
-                        pp.context[k] = v
-
-            for glob, defn in self._definition.iteritems():
-                if glob and not glob.startswith('-') and fnmatch(file, glob):
-                    apply_defn(defn)
-
-            for flag in flags:
-                apply_defn(self._definition.get(flag, {}))
-
-            pp.out = StringIO()
-            pp.do_include(file)
-            return 0, pp.out.getvalue(), ''
-
-        return 1, '', ''
-
-
-class TestFakeCompiler(unittest.TestCase):
-    def test_fake_compiler(self):
-        with MockedOpen({
-            'file': 'A B C',
-            'file.c': 'A B C',
-        }):
-            compiler = FakeCompiler({
-                'A': '1',
-                'B': '2',
-            })
-            self.assertEquals(compiler(None, ['-E', 'file']),
-                              (0, '1 2 C', ''))
-
-            compiler = FakeCompiler({
-                None: {
-                    'A': '1',
-                    'B': '2',
-                },
-                '-foo': {
-                    'C': 'foo',
-                },
-                '-bar': {
-                    'B': 'bar',
-                    'C': 'bar',
-                },
-                '-qux': {
-                    'B': False,
-                },
-                '*.c': {
-                    'B': '42',
-                },
-            })
-            self.assertEquals(compiler(None, ['-E', 'file']),
-                              (0, '1 2 C', ''))
-            self.assertEquals(compiler(None, ['-E', '-foo', 'file']),
-                              (0, '1 2 foo', ''))
-            self.assertEquals(compiler(None, ['-E', '-bar', 'file']),
-                              (0, '1 bar bar', ''))
-            self.assertEquals(compiler(None, ['-E', '-qux', 'file']),
-                              (0, '1 B C', ''))
-            self.assertEquals(compiler(None, ['-E', '-foo', '-bar', 'file']),
-                              (0, '1 bar bar', ''))
-            self.assertEquals(compiler(None, ['-E', '-bar', '-foo', 'file']),
-                              (0, '1 bar foo', ''))
-            self.assertEquals(compiler(None, ['-E', '-bar', '-qux', 'file']),
-                              (0, '1 B bar', ''))
-            self.assertEquals(compiler(None, ['-E', '-qux', '-bar', 'file']),
-                              (0, '1 bar bar', ''))
-            self.assertEquals(compiler(None, ['-E', 'file.c']),
-                              (0, '1 42 C', ''))
-            self.assertEquals(compiler(None, ['-E', '-bar', 'file.c']),
-                              (0, '1 bar bar', ''))
+from test_toolchain_helpers import FakeCompiler
 
 
 GCC_4_7 = FakeCompiler({
     None: {
         '__GNUC__': 4,
         '__GNUC_MINOR__': 7,
         '__GNUC_PATCHLEVEL__': 3,
         '__STDC__': 1,
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
@@ -0,0 +1,233 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import re
+import types
+import unittest
+
+from fnmatch import fnmatch
+from StringIO import StringIO
+from textwrap import dedent
+
+from mozunit import (
+    main,
+    MockedOpen,
+)
+
+from mozbuild.preprocessor import Preprocessor
+
+
+class CompilerPreprocessor(Preprocessor):
+    VARSUBST = re.compile('(?P<VAR>\w+)', re.U)
+    NON_WHITESPACE = re.compile('\S')
+
+    def __init__(self, *args, **kwargs):
+        Preprocessor.__init__(self, *args, **kwargs)
+        self.do_filter('c_substitution')
+        self.setMarker('#\s*')
+
+    def do_if(self, *args, **kwargs):
+        # The C preprocessor handles numbers following C rules, which is a
+        # different handling than what our Preprocessor does out of the box.
+        # Hack around it enough that the configure tests work properly.
+        context = self.context
+        def normalize_numbers(value):
+            if isinstance(value, types.StringTypes):
+                if value[-1:] == 'L' and value[:-1].isdigit():
+                    value = int(value[:-1])
+            return value
+        self.context = self.Context(
+            (k, normalize_numbers(v)) for k, v in context.iteritems()
+        )
+        try:
+            return Preprocessor.do_if(self, *args, **kwargs)
+        finally:
+            self.context = context
+
+    class Context(dict):
+        def __missing__(self, key):
+            return None
+
+    def filter_c_substitution(self, line):
+        def repl(matchobj):
+            varname = matchobj.group('VAR')
+            if varname in self.context:
+                result = str(self.context[varname])
+                # The C preprocessor inserts whitespaces around expanded
+                # symbols.
+                start, end = matchobj.span('VAR')
+                if self.NON_WHITESPACE.match(line[start-1:start]):
+                    result = ' ' + result
+                if self.NON_WHITESPACE.match(line[end:end+1]):
+                    result = result + ' '
+                return result
+            return matchobj.group(0)
+        return self.VARSUBST.sub(repl, line)
+
+
+class TestCompilerPreprocessor(unittest.TestCase):
+    def test_expansion(self):
+        pp = CompilerPreprocessor({
+            'A': 1,
+            'B': '2',
+            'C': 'c',
+        })
+        pp.out = StringIO()
+        input = StringIO('A.B.C')
+        input.name = 'foo'
+        pp.do_include(input)
+
+        self.assertEquals(pp.out.getvalue(), '1 . 2 . c')
+
+    def test_condition(self):
+        pp = CompilerPreprocessor({
+            'A': 1,
+            'B': '2',
+            'C': '0L',
+        })
+        pp.out = StringIO()
+        input = StringIO(dedent('''\
+            #ifdef A
+            IFDEF_A
+            #endif
+            #if A
+            IF_A
+            #endif
+            #  if B
+            IF_B
+            #  else
+            IF_NOT_B
+            #  endif
+            #if !C
+            IF_NOT_C
+            #else
+            IF_C
+            #endif
+        '''))
+        input.name = 'foo'
+        pp.do_include(input)
+
+        self.assertEquals('IFDEF_A\nIF_A\nIF_B\nIF_NOT_C\n', pp.out.getvalue())
+
+
+class FakeCompiler(object):
+    '''Defines a fake compiler for use in toolchain tests below.
+
+    The definition given when creating an instance can have one of two
+    forms:
+    - a dict giving preprocessor symbols and their respective value, e.g.
+        { '__GNUC__': 4, '__STDC__': 1 }
+    - a dict associating flags to preprocessor symbols. An entry for `None`
+      is required in this case. Those are the baseline preprocessor symbols.
+      Additional entries describe additional flags to set or existing flags
+      to unset (with a value of `False`).
+        {
+          None: { '__GNUC__': 4, '__STDC__': 1, '__STRICT_ANSI__': 1 },
+          '-std=gnu99': { '__STDC_VERSION__': '199901L',
+                          '__STRICT_ANSI__': False },
+        }
+      With the dict above, invoking the preprocessor with no additional flags
+      would define __GNUC__, __STDC__ and __STRICT_ANSI__, and with -std=gnu99,
+      __GNUC__, __STDC__, and __STDC_VERSION__ (__STRICT_ANSI__ would be
+      unset).
+      It is also possible to have different symbols depending on the source
+      file extension. In this case, the key is '*.ext'. e.g.
+        {
+          '*.c': { '__STDC__': 1 },
+          '*.cpp': { '__cplusplus': '199711L' },
+        }
+    '''
+    def __init__(self, definition):
+        if definition.get(None) is None:
+            definition = {None: definition}
+        self._definition = definition
+
+    def __call__(self, stdin, args):
+        files = [arg for arg in args if not arg.startswith('-')]
+        flags = [arg for arg in args if arg.startswith('-')]
+        if '-E' in flags:
+            assert len(files) == 1
+            file = files[0]
+            pp = CompilerPreprocessor(self._definition[None])
+
+            def apply_defn(defn):
+                for k, v in defn.iteritems():
+                    if v is False:
+                        if k in pp.context:
+                            del pp.context[k]
+                    else:
+                        pp.context[k] = v
+
+            for glob, defn in self._definition.iteritems():
+                if glob and not glob.startswith('-') and fnmatch(file, glob):
+                    apply_defn(defn)
+
+            for flag in flags:
+                apply_defn(self._definition.get(flag, {}))
+
+            pp.out = StringIO()
+            pp.do_include(file)
+            return 0, pp.out.getvalue(), ''
+
+        return 1, '', ''
+
+
+class TestFakeCompiler(unittest.TestCase):
+    def test_fake_compiler(self):
+        with MockedOpen({
+            'file': 'A B C',
+            'file.c': 'A B C',
+        }):
+            compiler = FakeCompiler({
+                'A': '1',
+                'B': '2',
+            })
+            self.assertEquals(compiler(None, ['-E', 'file']),
+                              (0, '1 2 C', ''))
+
+            compiler = FakeCompiler({
+                None: {
+                    'A': '1',
+                    'B': '2',
+                },
+                '-foo': {
+                    'C': 'foo',
+                },
+                '-bar': {
+                    'B': 'bar',
+                    'C': 'bar',
+                },
+                '-qux': {
+                    'B': False,
+                },
+                '*.c': {
+                    'B': '42',
+                },
+            })
+            self.assertEquals(compiler(None, ['-E', 'file']),
+                              (0, '1 2 C', ''))
+            self.assertEquals(compiler(None, ['-E', '-foo', 'file']),
+                              (0, '1 2 foo', ''))
+            self.assertEquals(compiler(None, ['-E', '-bar', 'file']),
+                              (0, '1 bar bar', ''))
+            self.assertEquals(compiler(None, ['-E', '-qux', 'file']),
+                              (0, '1 B C', ''))
+            self.assertEquals(compiler(None, ['-E', '-foo', '-bar', 'file']),
+                              (0, '1 bar bar', ''))
+            self.assertEquals(compiler(None, ['-E', '-bar', '-foo', 'file']),
+                              (0, '1 bar foo', ''))
+            self.assertEquals(compiler(None, ['-E', '-bar', '-qux', 'file']),
+                              (0, '1 B bar', ''))
+            self.assertEquals(compiler(None, ['-E', '-qux', '-bar', 'file']),
+                              (0, '1 bar bar', ''))
+            self.assertEquals(compiler(None, ['-E', 'file.c']),
+                              (0, '1 42 C', ''))
+            self.assertEquals(compiler(None, ['-E', '-bar', 'file.c']),
+                              (0, '1 bar bar', ''))
+
+
+if __name__ == '__main__':
+    main()