Bug 1270446 - Move CompilerPreprocessor, FakeCompiler and their tests to a separate file. r=chmanchester
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 22 Apr 2016 10:03:56 +0900
changeset 337396 8e38b67846f12fc5423810c67a1f25721f57cdc4
parent 337395 6faf0619a211d3841089aabb2405183483eb4a9a
child 337397 471bf0db284398af3ac4c83615e56ad22a0e2fe8
push id1183
push userraliiev@mozilla.com
push dateMon, 05 Sep 2016 20:01:49 +0000
treeherdermozilla-release@3148731bed45 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1270446
milestone49.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1270446 - Move CompilerPreprocessor, FakeCompiler and their tests to a separate file. r=chmanchester This makes the toolchain.configure tests more prominent in the file.
python/moz.build
python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
--- a/python/moz.build
+++ b/python/moz.build
@@ -37,16 +37,17 @@ PYTHON_UNIT_TESTS += [
     'mozbuild/mozbuild/test/backend/test_recursivemake.py',
     'mozbuild/mozbuild/test/backend/test_visualstudio.py',
     'mozbuild/mozbuild/test/compilation/test_warnings.py',
     'mozbuild/mozbuild/test/configure/test_checks_configure.py',
     'mozbuild/mozbuild/test/configure/test_configure.py',
     'mozbuild/mozbuild/test/configure/test_moz_configure.py',
     'mozbuild/mozbuild/test/configure/test_options.py',
     'mozbuild/mozbuild/test/configure/test_toolchain_configure.py',
+    'mozbuild/mozbuild/test/configure/test_toolchain_helpers.py',
     'mozbuild/mozbuild/test/configure/test_util.py',
     'mozbuild/mozbuild/test/controller/test_ccachestats.py',
     'mozbuild/mozbuild/test/controller/test_clobber.py',
     'mozbuild/mozbuild/test/frontend/test_context.py',
     'mozbuild/mozbuild/test/frontend/test_emitter.py',
     'mozbuild/mozbuild/test/frontend/test_namespaces.py',
     'mozbuild/mozbuild/test/frontend/test_reader.py',
     'mozbuild/mozbuild/test/frontend/test_sandbox.py',
--- a/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
@@ -1,242 +1,24 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
 import os
-import re
-import types
-import unittest
 
-from collections import defaultdict
-from fnmatch import fnmatch
 from StringIO import StringIO
-from textwrap import dedent
 
-from mozunit import (
-    main,
-    MockedOpen,
-)
+from mozunit import main
 
 from common import BaseConfigureTest
-from mozbuild.preprocessor import Preprocessor
 from mozpack import path as mozpath
-
-
-class CompilerPreprocessor(Preprocessor):
-    VARSUBST = re.compile('(?P<VAR>\w+)', re.U)
-    NON_WHITESPACE = re.compile('\S')
-
-    def __init__(self, *args, **kwargs):
-        Preprocessor.__init__(self, *args, **kwargs)
-        self.do_filter('c_substitution')
-        self.setMarker('#\s*')
-
-    def do_if(self, *args, **kwargs):
-        # The C preprocessor handles numbers following C rules, which is a
-        # different handling than what our Preprocessor does out of the box.
-        # Hack around it enough that the configure tests work properly.
-        context = self.context
-        def normalize_numbers(value):
-            if isinstance(value, types.StringTypes):
-                if value[-1:] == 'L' and value[:-1].isdigit():
-                    value = int(value[:-1])
-            return value
-        self.context = self.Context(
-            (k, normalize_numbers(v)) for k, v in context.iteritems()
-        )
-        try:
-            return Preprocessor.do_if(self, *args, **kwargs)
-        finally:
-            self.context = context
-
-    class Context(dict):
-        def __missing__(self, key):
-            return None
-
-    def filter_c_substitution(self, line):
-        def repl(matchobj):
-            varname = matchobj.group('VAR')
-            if varname in self.context:
-                result = str(self.context[varname])
-                # The C preprocessor inserts whitespaces around expanded
-                # symbols.
-                start, end = matchobj.span('VAR')
-                if self.NON_WHITESPACE.match(line[start-1:start]):
-                    result = ' ' + result
-                if self.NON_WHITESPACE.match(line[end:end+1]):
-                    result = result + ' '
-                return result
-            return matchobj.group(0)
-        return self.VARSUBST.sub(repl, line)
-
-
-class TestCompilerPreprocessor(unittest.TestCase):
-    def test_expansion(self):
-        pp = CompilerPreprocessor({
-            'A': 1,
-            'B': '2',
-            'C': 'c',
-        })
-        pp.out = StringIO()
-        input = StringIO('A.B.C')
-        input.name = 'foo'
-        pp.do_include(input)
-
-        self.assertEquals(pp.out.getvalue(), '1 . 2 . c')
-
-    def test_condition(self):
-        pp = CompilerPreprocessor({
-            'A': 1,
-            'B': '2',
-            'C': '0L',
-        })
-        pp.out = StringIO()
-        input = StringIO(dedent('''\
-            #ifdef A
-            IFDEF_A
-            #endif
-            #if A
-            IF_A
-            #endif
-            #  if B
-            IF_B
-            #  else
-            IF_NOT_B
-            #  endif
-            #if !C
-            IF_NOT_C
-            #else
-            IF_C
-            #endif
-        '''))
-        input.name = 'foo'
-        pp.do_include(input)
-
-        self.assertEquals('IFDEF_A\nIF_A\nIF_B\nIF_NOT_C\n', pp.out.getvalue())
-
-
-class FakeCompiler(object):
-    '''Defines a fake compiler for use in toolchain tests below.
-
-    The definition given when creating an instance can have one of two
-    forms:
-    - a dict giving preprocessor symbols and their respective value, e.g.
-        { '__GNUC__': 4, '__STDC__': 1 }
-    - a dict associating flags to preprocessor symbols. An entry for `None`
-      is required in this case. Those are the baseline preprocessor symbols.
-      Additional entries describe additional flags to set or existing flags
-      to unset (with a value of `False`).
-        {
-          None: { '__GNUC__': 4, '__STDC__': 1, '__STRICT_ANSI__': 1 },
-          '-std=gnu99': { '__STDC_VERSION__': '199901L',
-                          '__STRICT_ANSI__': False },
-        }
-      With the dict above, invoking the preprocessor with no additional flags
-      would define __GNUC__, __STDC__ and __STRICT_ANSI__, and with -std=gnu99,
-      __GNUC__, __STDC__, and __STDC_VERSION__ (__STRICT_ANSI__ would be
-      unset).
-      It is also possible to have different symbols depending on the source
-      file extension. In this case, the key is '*.ext'. e.g.
-        {
-          '*.c': { '__STDC__': 1 },
-          '*.cpp': { '__cplusplus': '199711L' },
-        }
-    '''
-    def __init__(self, definition):
-        if definition.get(None) is None:
-            definition = {None: definition}
-        self._definition = definition
-
-    def __call__(self, stdin, args):
-        files = [arg for arg in args if not arg.startswith('-')]
-        flags = [arg for arg in args if arg.startswith('-')]
-        if '-E' in flags:
-            assert len(files) == 1
-            file = files[0]
-            pp = CompilerPreprocessor(self._definition[None])
-
-            def apply_defn(defn):
-                for k, v in defn.iteritems():
-                    if v is False:
-                        if k in pp.context:
-                            del pp.context[k]
-                    else:
-                        pp.context[k] = v
-
-            for glob, defn in self._definition.iteritems():
-                if glob and not glob.startswith('-') and fnmatch(file, glob):
-                    apply_defn(defn)
-
-            for flag in flags:
-                apply_defn(self._definition.get(flag, {}))
-
-            pp.out = StringIO()
-            pp.do_include(file)
-            return 0, pp.out.getvalue(), ''
-
-        return 1, '', ''
-
-
-class TestFakeCompiler(unittest.TestCase):
-    def test_fake_compiler(self):
-        with MockedOpen({
-            'file': 'A B C',
-            'file.c': 'A B C',
-        }):
-            compiler = FakeCompiler({
-                'A': '1',
-                'B': '2',
-            })
-            self.assertEquals(compiler(None, ['-E', 'file']),
-                              (0, '1 2 C', ''))
-
-            compiler = FakeCompiler({
-                None: {
-                    'A': '1',
-                    'B': '2',
-                },
-                '-foo': {
-                    'C': 'foo',
-                },
-                '-bar': {
-                    'B': 'bar',
-                    'C': 'bar',
-                },
-                '-qux': {
-                    'B': False,
-                },
-                '*.c': {
-                    'B': '42',
-                },
-            })
-            self.assertEquals(compiler(None, ['-E', 'file']),
-                              (0, '1 2 C', ''))
-            self.assertEquals(compiler(None, ['-E', '-foo', 'file']),
-                              (0, '1 2 foo', ''))
-            self.assertEquals(compiler(None, ['-E', '-bar', 'file']),
-                              (0, '1 bar bar', ''))
-            self.assertEquals(compiler(None, ['-E', '-qux', 'file']),
-                              (0, '1 B C', ''))
-            self.assertEquals(compiler(None, ['-E', '-foo', '-bar', 'file']),
-                              (0, '1 bar bar', ''))
-            self.assertEquals(compiler(None, ['-E', '-bar', '-foo', 'file']),
-                              (0, '1 bar foo', ''))
-            self.assertEquals(compiler(None, ['-E', '-bar', '-qux', 'file']),
-                              (0, '1 B bar', ''))
-            self.assertEquals(compiler(None, ['-E', '-qux', '-bar', 'file']),
-                              (0, '1 bar bar', ''))
-            self.assertEquals(compiler(None, ['-E', 'file.c']),
-                              (0, '1 42 C', ''))
-            self.assertEquals(compiler(None, ['-E', '-bar', 'file.c']),
-                              (0, '1 bar bar', ''))
+from test_toolchain_helpers import FakeCompiler
 
 
 GCC_4_7 = FakeCompiler({
     None: {
         '__GNUC__': 4,
         '__GNUC_MINOR__': 7,
         '__GNUC_PATCHLEVEL__': 3,
         '__STDC__': 1,
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
@@ -0,0 +1,233 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import re
+import types
+import unittest
+
+from fnmatch import fnmatch
+from StringIO import StringIO
+from textwrap import dedent
+
+from mozunit import (
+    main,
+    MockedOpen,
+)
+
+from mozbuild.preprocessor import Preprocessor
+
+
+class CompilerPreprocessor(Preprocessor):
+    VARSUBST = re.compile('(?P<VAR>\w+)', re.U)
+    NON_WHITESPACE = re.compile('\S')
+
+    def __init__(self, *args, **kwargs):
+        Preprocessor.__init__(self, *args, **kwargs)
+        self.do_filter('c_substitution')
+        self.setMarker('#\s*')
+
+    def do_if(self, *args, **kwargs):
+        # The C preprocessor handles numbers following C rules, which is a
+        # different handling than what our Preprocessor does out of the box.
+        # Hack around it enough that the configure tests work properly.
+        context = self.context
+        def normalize_numbers(value):
+            if isinstance(value, types.StringTypes):
+                if value[-1:] == 'L' and value[:-1].isdigit():
+                    value = int(value[:-1])
+            return value
+        self.context = self.Context(
+            (k, normalize_numbers(v)) for k, v in context.iteritems()
+        )
+        try:
+            return Preprocessor.do_if(self, *args, **kwargs)
+        finally:
+            self.context = context
+
+    class Context(dict):
+        def __missing__(self, key):
+            return None
+
+    def filter_c_substitution(self, line):
+        def repl(matchobj):
+            varname = matchobj.group('VAR')
+            if varname in self.context:
+                result = str(self.context[varname])
+                # The C preprocessor inserts whitespaces around expanded
+                # symbols.
+                start, end = matchobj.span('VAR')
+                if self.NON_WHITESPACE.match(line[start-1:start]):
+                    result = ' ' + result
+                if self.NON_WHITESPACE.match(line[end:end+1]):
+                    result = result + ' '
+                return result
+            return matchobj.group(0)
+        return self.VARSUBST.sub(repl, line)
+
+
+class TestCompilerPreprocessor(unittest.TestCase):
+    def test_expansion(self):
+        pp = CompilerPreprocessor({
+            'A': 1,
+            'B': '2',
+            'C': 'c',
+        })
+        pp.out = StringIO()
+        input = StringIO('A.B.C')
+        input.name = 'foo'
+        pp.do_include(input)
+
+        self.assertEquals(pp.out.getvalue(), '1 . 2 . c')
+
+    def test_condition(self):
+        pp = CompilerPreprocessor({
+            'A': 1,
+            'B': '2',
+            'C': '0L',
+        })
+        pp.out = StringIO()
+        input = StringIO(dedent('''\
+            #ifdef A
+            IFDEF_A
+            #endif
+            #if A
+            IF_A
+            #endif
+            #  if B
+            IF_B
+            #  else
+            IF_NOT_B
+            #  endif
+            #if !C
+            IF_NOT_C
+            #else
+            IF_C
+            #endif
+        '''))
+        input.name = 'foo'
+        pp.do_include(input)
+
+        self.assertEquals('IFDEF_A\nIF_A\nIF_B\nIF_NOT_C\n', pp.out.getvalue())
+
+
+class FakeCompiler(object):
+    '''Defines a fake compiler for use in toolchain tests below.
+
+    The definition given when creating an instance can have one of two
+    forms:
+    - a dict giving preprocessor symbols and their respective value, e.g.
+        { '__GNUC__': 4, '__STDC__': 1 }
+    - a dict associating flags to preprocessor symbols. An entry for `None`
+      is required in this case. Those are the baseline preprocessor symbols.
+      Additional entries describe additional flags to set or existing flags
+      to unset (with a value of `False`).
+        {
+          None: { '__GNUC__': 4, '__STDC__': 1, '__STRICT_ANSI__': 1 },
+          '-std=gnu99': { '__STDC_VERSION__': '199901L',
+                          '__STRICT_ANSI__': False },
+        }
+      With the dict above, invoking the preprocessor with no additional flags
+      would define __GNUC__, __STDC__ and __STRICT_ANSI__, and with -std=gnu99,
+      __GNUC__, __STDC__, and __STDC_VERSION__ (__STRICT_ANSI__ would be
+      unset).
+      It is also possible to have different symbols depending on the source
+      file extension. In this case, the key is '*.ext'. e.g.
+        {
+          '*.c': { '__STDC__': 1 },
+          '*.cpp': { '__cplusplus': '199711L' },
+        }
+    '''
+    def __init__(self, definition):
+        if definition.get(None) is None:
+            definition = {None: definition}
+        self._definition = definition
+
+    def __call__(self, stdin, args):
+        files = [arg for arg in args if not arg.startswith('-')]
+        flags = [arg for arg in args if arg.startswith('-')]
+        if '-E' in flags:
+            assert len(files) == 1
+            file = files[0]
+            pp = CompilerPreprocessor(self._definition[None])
+
+            def apply_defn(defn):
+                for k, v in defn.iteritems():
+                    if v is False:
+                        if k in pp.context:
+                            del pp.context[k]
+                    else:
+                        pp.context[k] = v
+
+            for glob, defn in self._definition.iteritems():
+                if glob and not glob.startswith('-') and fnmatch(file, glob):
+                    apply_defn(defn)
+
+            for flag in flags:
+                apply_defn(self._definition.get(flag, {}))
+
+            pp.out = StringIO()
+            pp.do_include(file)
+            return 0, pp.out.getvalue(), ''
+
+        return 1, '', ''
+
+
+class TestFakeCompiler(unittest.TestCase):
+    def test_fake_compiler(self):
+        with MockedOpen({
+            'file': 'A B C',
+            'file.c': 'A B C',
+        }):
+            compiler = FakeCompiler({
+                'A': '1',
+                'B': '2',
+            })
+            self.assertEquals(compiler(None, ['-E', 'file']),
+                              (0, '1 2 C', ''))
+
+            compiler = FakeCompiler({
+                None: {
+                    'A': '1',
+                    'B': '2',
+                },
+                '-foo': {
+                    'C': 'foo',
+                },
+                '-bar': {
+                    'B': 'bar',
+                    'C': 'bar',
+                },
+                '-qux': {
+                    'B': False,
+                },
+                '*.c': {
+                    'B': '42',
+                },
+            })
+            self.assertEquals(compiler(None, ['-E', 'file']),
+                              (0, '1 2 C', ''))
+            self.assertEquals(compiler(None, ['-E', '-foo', 'file']),
+                              (0, '1 2 foo', ''))
+            self.assertEquals(compiler(None, ['-E', '-bar', 'file']),
+                              (0, '1 bar bar', ''))
+            self.assertEquals(compiler(None, ['-E', '-qux', 'file']),
+                              (0, '1 B C', ''))
+            self.assertEquals(compiler(None, ['-E', '-foo', '-bar', 'file']),
+                              (0, '1 bar bar', ''))
+            self.assertEquals(compiler(None, ['-E', '-bar', '-foo', 'file']),
+                              (0, '1 bar foo', ''))
+            self.assertEquals(compiler(None, ['-E', '-bar', '-qux', 'file']),
+                              (0, '1 B bar', ''))
+            self.assertEquals(compiler(None, ['-E', '-qux', '-bar', 'file']),
+                              (0, '1 bar bar', ''))
+            self.assertEquals(compiler(None, ['-E', 'file.c']),
+                              (0, '1 42 C', ''))
+            self.assertEquals(compiler(None, ['-E', '-bar', 'file.c']),
+                              (0, '1 bar bar', ''))
+
+
+if __name__ == '__main__':
+    main()