author | Gregory Szorc <gps@mozilla.com> |
Tue, 29 Jan 2013 06:24:24 -0800 | |
changeset 130136 | 5cc7c40382089a61e15a8b691e70c5f6c3b63128 |
parent 129977 | 9f22692e440433ca23980d28669826bde51436f5 |
child 130137 | 7c73b5af624735e5042a40086b45602f3357c0ed |
push id | 2323 |
push user | bbajaj@mozilla.com |
push date | Mon, 01 Apr 2013 19:47:02 +0000 |
treeherder | mozilla-beta@7712be144d91 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ted |
bugs | 784841 |
milestone | 21.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/build/ConfigStatus.py +++ b/build/ConfigStatus.py @@ -1,228 +1,32 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Combined with build/autoconf/config.status.m4, ConfigStatus is an almost # drop-in replacement for autoconf 2.13's config.status, with features # borrowed from autoconf > 2.5, and additional features. -from __future__ import with_statement +import os +import sys + from optparse import OptionParser -import sys, re, os, posixpath, ntpath -import errno -from StringIO import StringIO -from os.path import relpath -# Standalone js doesn't have virtualenv. -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'config')) +from mozbuild.backend.configenvironment import ConfigEnvironment + from Preprocessor import Preprocessor # Basic logging facility verbose = False def log(string): if verbose: print >>sys.stderr, string -def ensureParentDir(file): - '''Ensures the directory parent to the given file exists''' - dir = os.path.dirname(file) - if dir and not os.path.exists(dir): - try: - os.makedirs(dir) - except OSError, error: - if error.errno != errno.EEXIST: - raise - -class FileAvoidWrite(StringIO): - '''file-like object that buffers its output and only writes it to disk - if the new contents are different from what the file may already contain. - ''' - def __init__(self, filename): - self.filename = filename - StringIO.__init__(self) - - def close(self): - buf = self.getvalue() - StringIO.close(self) - try: - file = open(self.filename, 'rU') - except IOError: - pass - else: - try: - if file.read() == buf: - log("%s is unchanged" % relpath(self.filename, os.curdir)) - return - except IOError: - pass - finally: - file.close() - - log("creating %s" % relpath(self.filename, os.curdir)) - ensureParentDir(self.filename) - with open(self.filename, 'w') as file: - file.write(buf) - - def __enter__(self): - return self - def __exit__(self, type, value, traceback): - self.close() - -def shell_escape(s): - '''Escape some characters with a backslash, and double dollar signs. - ''' - return re.sub('''([ \t`#$^&*(){}\\|;'"<>?\[\]])''', r'\\\1', str(s)).replace('$', '$$') - -class ConfigEnvironment(object): - '''A ConfigEnvironment is defined by a source directory and a build - directory. It preprocesses files from the source directory and stores - the result in the object directory. - - There are two types of files: config files and config headers, - each treated through a different member function. - - Creating a ConfigEnvironment requires a few arguments: - - topsrcdir and topobjdir are, respectively, the top source and - the top object directory. - - defines is a list of (name, value) tuples. In autoconf, these are - set with AC_DEFINE and AC_DEFINE_UNQUOTED - - non_global_defines are a list of names appearing in defines above - that are not meant to be exported in ACDEFINES and ALLDEFINES (see - below) - - substs is a list of (name, value) tuples. In autoconf, these are - set with AC_SUBST. - - ConfigEnvironment automatically defines two additional substs variables - from all the defines not appearing in non_global_defines: - - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on - preprocessor command lines. The order in which defines were given - when creating the ConfigEnvironment is preserved. - - ALLDEFINES contains the defines in the form #define NAME VALUE, in - sorted order, for use in config files, for an automatic listing of - defines. - and another additional subst variable from all the other substs: - - ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted - order, for use in autoconf.mk. It includes ACDEFINES, but doesn't - include ALLDEFINES. - - ConfigEnvironment expects a "top_srcdir" subst to be set with the top - source directory, in msys format on windows. It is used to derive a - "srcdir" subst when treating config files. It can either be an absolute - path or a path relative to the topobjdir. - ''' - - def __init__(self, topobjdir = '.', topsrcdir = '.', - defines = [], non_global_defines = [], substs = []): - self.defines = dict(defines) - self.substs = dict(substs) - self.topsrcdir = topsrcdir - self.topobjdir = topobjdir - global_defines = [name for name, value in defines if not name in non_global_defines] - self.substs['ACDEFINES'] = ' '.join(["-D%s=%s" % (name, shell_escape(self.defines[name])) for name in global_defines]) - self.substs['ALLSUBSTS'] = '\n'.join(sorted(["%s = %s" % (name, self.substs[name]) for name in self.substs])) - self.substs['ALLDEFINES'] = '\n'.join(sorted(["#define %s %s" % (name, self.defines[name]) for name in global_defines])) - - def get_relative_srcdir(self, file): - '''Returns the relative source directory for the given file, always - using / as a path separator. - ''' - assert(isinstance(file, basestring)) - dir = posixpath.dirname(relpath(file, self.topobjdir).replace(os.sep, '/')) - if dir: - return dir - return '.' - - def get_top_srcdir(self, file): - '''Returns a normalized top_srcdir for the given file: if - substs['top_srcdir'] is a relative path, it is relative to the - topobjdir. Adjust it to be relative to the file path.''' - top_srcdir = self.substs['top_srcdir'] - if posixpath.isabs(top_srcdir) or ntpath.isabs(top_srcdir): - return top_srcdir - return posixpath.normpath(posixpath.join(self.get_depth(file), top_srcdir)) - - def get_file_srcdir(self, file): - '''Returns the srcdir for the given file, where srcdir is in msys - format on windows, thus derived from top_srcdir. - ''' - dir = self.get_relative_srcdir(file) - top_srcdir = self.get_top_srcdir(file) - return posixpath.normpath(posixpath.join(top_srcdir, dir)) - - def get_depth(self, file): - '''Returns the DEPTH for the given file, that is, the path to the - object directory relative to the directory containing the given file. - Always uses / as a path separator. - ''' - return relpath(self.topobjdir, os.path.dirname(file)).replace(os.sep, '/') - - def get_input(self, file): - '''Returns the input file path in the source tree that can be used - to create the given config file or header. - ''' - assert(isinstance(file, basestring)) - return os.path.normpath(os.path.join(self.topsrcdir, "%s.in" % relpath(file, self.topobjdir))) - - def create_config_file(self, path): - '''Creates the given config file. A config file is generated by - taking the corresponding source file and replacing occurences of - "@VAR@" by the value corresponding to "VAR" in the substs dict. - - Additional substs are defined according to the file being treated: - "srcdir" for its the path to its source directory - "relativesrcdir" for its source directory relative to the top - "DEPTH" for the path to the top object directory - ''' - input = self.get_input(path) - pp = Preprocessor() - pp.context.update(self.substs) - pp.context.update(top_srcdir = self.get_top_srcdir(path)) - pp.context.update(srcdir = self.get_file_srcdir(path)) - pp.context.update(relativesrcdir = self.get_relative_srcdir(path)) - pp.context.update(DEPTH = self.get_depth(path)) - pp.do_filter('attemptSubstitution') - pp.setMarker(None) - with FileAvoidWrite(path) as pp.out: - pp.do_include(input) - - def create_config_header(self, path): - '''Creates the given config header. A config header is generated by - taking the corresponding source file and replacing some #define/#undef - occurences: - "#undef NAME" is turned into "#define NAME VALUE" - "#define NAME" is unchanged - "#define NAME ORIGINAL_VALUE" is turned into "#define NAME VALUE" - "#undef UNKNOWN_NAME" is turned into "/* #undef UNKNOWN_NAME */" - Whitespaces are preserved. - ''' - with open(self.get_input(path), 'rU') as input: - ensureParentDir(path) - output = FileAvoidWrite(path) - r = re.compile('^\s*#\s*(?P<cmd>[a-z]+)(?:\s+(?P<name>\S+)(?:\s+(?P<value>\S+))?)?', re.U) - for l in input: - m = r.match(l) - if m: - cmd = m.group('cmd') - name = m.group('name') - value = m.group('value') - if name: - if name in self.defines: - if cmd == 'define' and value: - l = l[:m.start('value')] + str(self.defines[name]) + l[m.end('value'):] - elif cmd == 'undef': - l = l[:m.start('cmd')] + 'define' + l[m.end('cmd'):m.end('name')] + ' ' + str(self.defines[name]) + l[m.end('name'):] - elif cmd == 'undef': - l = '/* ' + l[:m.end('name')] + ' */' + l[m.end('name'):] - - output.write(l) - output.close() - def config_status(topobjdir = '.', topsrcdir = '.', defines = [], non_global_defines = [], substs = [], files = [], headers = []): '''Main function, providing config.status functionality. Contrary to config.status, it doesn't use CONFIG_FILES or CONFIG_HEADERS variables, but like config.status from autoconf 2.6, single files may be generated with the --file and --header options. Several such options can @@ -271,19 +75,18 @@ def config_status(topobjdir = '.', topsr parser.add_option('-n', dest='not_topobjdir', action='store_true', help='do not consider current directory as top object directory') (options, args) = parser.parse_args() # Without -n, the current directory is meant to be the top object directory if not options.not_topobjdir: topobjdir = '.' - env = ConfigEnvironment(topobjdir = topobjdir, topsrcdir = topsrcdir, - defines = defines, non_global_defines = non_global_defines, - substs = substs) + env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines, + non_global_defines=non_global_defines, substs=substs) if options.recheck: # Execute configure from the top object directory if not os.path.isabs(topsrcdir): topsrcdir = relpath(topsrcdir, topobjdir) os.chdir(topobjdir) os.execlp('sh', 'sh', '-c', ' '.join([os.path.join(topsrcdir, 'configure'), env.substs['ac_configure_args'], '--no-create', '--no-recursion']))
--- a/build/Makefile.in +++ b/build/Makefile.in @@ -98,17 +98,16 @@ endif # Put a useful .gdbinit in the bin directory, to be picked up automatically # by GDB when we debug executables there. # NOTE: Keep .gdbinit in the topsrcdir for people who run gdb from the topsrcdir. GDBINIT_FILES := $(topsrcdir)/.gdbinit GDBINIT_DEST = $(FINAL_TARGET) INSTALL_TARGETS += GDBINIT PYTHON_UNIT_TESTS := \ - tests/unit-ConfigStatus.py \ tests/test.py \ $(NULL) include $(topsrcdir)/config/rules.mk # we install to _leaktest/ TARGET_DEPTH = .. include $(srcdir)/automation-build.mk
--- a/js/src/build/ConfigStatus.py +++ b/js/src/build/ConfigStatus.py @@ -1,228 +1,32 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Combined with build/autoconf/config.status.m4, ConfigStatus is an almost # drop-in replacement for autoconf 2.13's config.status, with features # borrowed from autoconf > 2.5, and additional features. -from __future__ import with_statement +import os +import sys + from optparse import OptionParser -import sys, re, os, posixpath, ntpath -import errno -from StringIO import StringIO -from os.path import relpath -# Standalone js doesn't have virtualenv. -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'config')) +from mozbuild.backend.configenvironment import ConfigEnvironment + from Preprocessor import Preprocessor # Basic logging facility verbose = False def log(string): if verbose: print >>sys.stderr, string -def ensureParentDir(file): - '''Ensures the directory parent to the given file exists''' - dir = os.path.dirname(file) - if dir and not os.path.exists(dir): - try: - os.makedirs(dir) - except OSError, error: - if error.errno != errno.EEXIST: - raise - -class FileAvoidWrite(StringIO): - '''file-like object that buffers its output and only writes it to disk - if the new contents are different from what the file may already contain. - ''' - def __init__(self, filename): - self.filename = filename - StringIO.__init__(self) - - def close(self): - buf = self.getvalue() - StringIO.close(self) - try: - file = open(self.filename, 'rU') - except IOError: - pass - else: - try: - if file.read() == buf: - log("%s is unchanged" % relpath(self.filename, os.curdir)) - return - except IOError: - pass - finally: - file.close() - - log("creating %s" % relpath(self.filename, os.curdir)) - ensureParentDir(self.filename) - with open(self.filename, 'w') as file: - file.write(buf) - - def __enter__(self): - return self - def __exit__(self, type, value, traceback): - self.close() - -def shell_escape(s): - '''Escape some characters with a backslash, and double dollar signs. - ''' - return re.sub('''([ \t`#$^&*(){}\\|;'"<>?\[\]])''', r'\\\1', str(s)).replace('$', '$$') - -class ConfigEnvironment(object): - '''A ConfigEnvironment is defined by a source directory and a build - directory. It preprocesses files from the source directory and stores - the result in the object directory. - - There are two types of files: config files and config headers, - each treated through a different member function. - - Creating a ConfigEnvironment requires a few arguments: - - topsrcdir and topobjdir are, respectively, the top source and - the top object directory. - - defines is a list of (name, value) tuples. In autoconf, these are - set with AC_DEFINE and AC_DEFINE_UNQUOTED - - non_global_defines are a list of names appearing in defines above - that are not meant to be exported in ACDEFINES and ALLDEFINES (see - below) - - substs is a list of (name, value) tuples. In autoconf, these are - set with AC_SUBST. - - ConfigEnvironment automatically defines two additional substs variables - from all the defines not appearing in non_global_defines: - - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on - preprocessor command lines. The order in which defines were given - when creating the ConfigEnvironment is preserved. - - ALLDEFINES contains the defines in the form #define NAME VALUE, in - sorted order, for use in config files, for an automatic listing of - defines. - and another additional subst variable from all the other substs: - - ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted - order, for use in autoconf.mk. It includes ACDEFINES, but doesn't - include ALLDEFINES. - - ConfigEnvironment expects a "top_srcdir" subst to be set with the top - source directory, in msys format on windows. It is used to derive a - "srcdir" subst when treating config files. It can either be an absolute - path or a path relative to the topobjdir. - ''' - - def __init__(self, topobjdir = '.', topsrcdir = '.', - defines = [], non_global_defines = [], substs = []): - self.defines = dict(defines) - self.substs = dict(substs) - self.topsrcdir = topsrcdir - self.topobjdir = topobjdir - global_defines = [name for name, value in defines if not name in non_global_defines] - self.substs['ACDEFINES'] = ' '.join(["-D%s=%s" % (name, shell_escape(self.defines[name])) for name in global_defines]) - self.substs['ALLSUBSTS'] = '\n'.join(sorted(["%s = %s" % (name, self.substs[name]) for name in self.substs])) - self.substs['ALLDEFINES'] = '\n'.join(sorted(["#define %s %s" % (name, self.defines[name]) for name in global_defines])) - - def get_relative_srcdir(self, file): - '''Returns the relative source directory for the given file, always - using / as a path separator. - ''' - assert(isinstance(file, basestring)) - dir = posixpath.dirname(relpath(file, self.topobjdir).replace(os.sep, '/')) - if dir: - return dir - return '.' - - def get_top_srcdir(self, file): - '''Returns a normalized top_srcdir for the given file: if - substs['top_srcdir'] is a relative path, it is relative to the - topobjdir. Adjust it to be relative to the file path.''' - top_srcdir = self.substs['top_srcdir'] - if posixpath.isabs(top_srcdir) or ntpath.isabs(top_srcdir): - return top_srcdir - return posixpath.normpath(posixpath.join(self.get_depth(file), top_srcdir)) - - def get_file_srcdir(self, file): - '''Returns the srcdir for the given file, where srcdir is in msys - format on windows, thus derived from top_srcdir. - ''' - dir = self.get_relative_srcdir(file) - top_srcdir = self.get_top_srcdir(file) - return posixpath.normpath(posixpath.join(top_srcdir, dir)) - - def get_depth(self, file): - '''Returns the DEPTH for the given file, that is, the path to the - object directory relative to the directory containing the given file. - Always uses / as a path separator. - ''' - return relpath(self.topobjdir, os.path.dirname(file)).replace(os.sep, '/') - - def get_input(self, file): - '''Returns the input file path in the source tree that can be used - to create the given config file or header. - ''' - assert(isinstance(file, basestring)) - return os.path.normpath(os.path.join(self.topsrcdir, "%s.in" % relpath(file, self.topobjdir))) - - def create_config_file(self, path): - '''Creates the given config file. A config file is generated by - taking the corresponding source file and replacing occurences of - "@VAR@" by the value corresponding to "VAR" in the substs dict. - - Additional substs are defined according to the file being treated: - "srcdir" for its the path to its source directory - "relativesrcdir" for its source directory relative to the top - "DEPTH" for the path to the top object directory - ''' - input = self.get_input(path) - pp = Preprocessor() - pp.context.update(self.substs) - pp.context.update(top_srcdir = self.get_top_srcdir(path)) - pp.context.update(srcdir = self.get_file_srcdir(path)) - pp.context.update(relativesrcdir = self.get_relative_srcdir(path)) - pp.context.update(DEPTH = self.get_depth(path)) - pp.do_filter('attemptSubstitution') - pp.setMarker(None) - with FileAvoidWrite(path) as pp.out: - pp.do_include(input) - - def create_config_header(self, path): - '''Creates the given config header. A config header is generated by - taking the corresponding source file and replacing some #define/#undef - occurences: - "#undef NAME" is turned into "#define NAME VALUE" - "#define NAME" is unchanged - "#define NAME ORIGINAL_VALUE" is turned into "#define NAME VALUE" - "#undef UNKNOWN_NAME" is turned into "/* #undef UNKNOWN_NAME */" - Whitespaces are preserved. - ''' - with open(self.get_input(path), 'rU') as input: - ensureParentDir(path) - output = FileAvoidWrite(path) - r = re.compile('^\s*#\s*(?P<cmd>[a-z]+)(?:\s+(?P<name>\S+)(?:\s+(?P<value>\S+))?)?', re.U) - for l in input: - m = r.match(l) - if m: - cmd = m.group('cmd') - name = m.group('name') - value = m.group('value') - if name: - if name in self.defines: - if cmd == 'define' and value: - l = l[:m.start('value')] + str(self.defines[name]) + l[m.end('value'):] - elif cmd == 'undef': - l = l[:m.start('cmd')] + 'define' + l[m.end('cmd'):m.end('name')] + ' ' + str(self.defines[name]) + l[m.end('name'):] - elif cmd == 'undef': - l = '/* ' + l[:m.end('name')] + ' */' + l[m.end('name'):] - - output.write(l) - output.close() - def config_status(topobjdir = '.', topsrcdir = '.', defines = [], non_global_defines = [], substs = [], files = [], headers = []): '''Main function, providing config.status functionality. Contrary to config.status, it doesn't use CONFIG_FILES or CONFIG_HEADERS variables, but like config.status from autoconf 2.6, single files may be generated with the --file and --header options. Several such options can @@ -271,19 +75,18 @@ def config_status(topobjdir = '.', topsr parser.add_option('-n', dest='not_topobjdir', action='store_true', help='do not consider current directory as top object directory') (options, args) = parser.parse_args() # Without -n, the current directory is meant to be the top object directory if not options.not_topobjdir: topobjdir = '.' - env = ConfigEnvironment(topobjdir = topobjdir, topsrcdir = topsrcdir, - defines = defines, non_global_defines = non_global_defines, - substs = substs) + env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines, + non_global_defines=non_global_defines, substs=substs) if options.recheck: # Execute configure from the top object directory if not os.path.isabs(topsrcdir): topsrcdir = relpath(topsrcdir, topobjdir) os.chdir(topobjdir) os.execlp('sh', 'sh', '-c', ' '.join([os.path.join(topsrcdir, 'configure'), env.substs['ac_configure_args'], '--no-create', '--no-recursion']))
--- a/python/Makefile.in +++ b/python/Makefile.in @@ -6,16 +6,17 @@ DEPTH := @DEPTH@ topsrcdir := @top_srcdir@ srcdir := @srcdir@ VPATH = @srcdir@ include $(DEPTH)/config/autoconf.mk test_dirs := \ mozbuild/mozbuild/test \ + mozbuild/mozbuild/test/backend \ mozbuild/mozbuild/test/compilation \ mozbuild/mozbuild/test/frontend \ mozbuild/mozpack/test \ $(NULL) PYTHON_UNIT_TESTS := $(foreach dir,$(test_dirs),$(wildcard $(srcdir)/$(dir)/*.py)) include $(topsrcdir)/config/rules.mk
new file mode 100644 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/configenvironment.py @@ -0,0 +1,179 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import ntpath +import os +import posixpath +import re + +from os.path import relpath + +from Preprocessor import Preprocessor + +from ..util import ( + ensureParentDir, + FileAvoidWrite, +) + + +RE_SHELL_ESCAPE = re.compile('''([ \t`#$^&*(){}\\|;'"<>?\[\]])''') + + +def shell_escape(s): + """Escape some characters with a backslash, and double dollar signs.""" + return RE_SHELL_ESCAPE.sub(r'\\\1', str(s)).replace('$', '$$') + + +class ConfigEnvironment(object): + """Perform actions associated with a configured but bare objdir. + + The purpose of this class is to preprocess files from the source directory + and output results in the object directory. + + There are two types of files: config files and config headers, + each treated through a different member function. + + Creating a ConfigEnvironment requires a few arguments: + - topsrcdir and topobjdir are, respectively, the top source and + the top object directory. + - defines is a list of (name, value) tuples. In autoconf, these are + set with AC_DEFINE and AC_DEFINE_UNQUOTED + - non_global_defines are a list of names appearing in defines above + that are not meant to be exported in ACDEFINES and ALLDEFINES (see + below) + - substs is a list of (name, value) tuples. In autoconf, these are + set with AC_SUBST. + + ConfigEnvironment automatically defines two additional substs variables + from all the defines not appearing in non_global_defines: + - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on + preprocessor command lines. The order in which defines were given + when creating the ConfigEnvironment is preserved. + - ALLDEFINES contains the defines in the form #define NAME VALUE, in + sorted order, for use in config files, for an automatic listing of + defines. + and another additional subst variable from all the other substs: + - ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted + order, for use in autoconf.mk. It includes ACDEFINES, but doesn't + include ALLDEFINES. + + ConfigEnvironment expects a "top_srcdir" subst to be set with the top + source directory, in msys format on windows. It is used to derive a + "srcdir" subst when treating config files. It can either be an absolute + path or a path relative to the topobjdir. + """ + + def __init__(self, topsrcdir, topobjdir, defines=[], non_global_defines=[], + substs=[]): + + self.defines = dict(defines) + self.substs = dict(substs) + self.topsrcdir = topsrcdir + self.topobjdir = topobjdir + global_defines = [name for name, value in defines + if not name in non_global_defines] + self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name, + shell_escape(self.defines[name])) for name in global_defines]) + self.substs['ALLSUBSTS'] = '\n'.join(sorted(['%s = %s' % (name, + self.substs[name]) for name in self.substs])) + self.substs['ALLDEFINES'] = '\n'.join(sorted(['#define %s %s' % (name, + self.defines[name]) for name in global_defines])) + + def get_relative_srcdir(self, file): + '''Returns the relative source directory for the given file, always + using / as a path separator. + ''' + assert(isinstance(file, basestring)) + dir = posixpath.dirname(relpath(file, self.topobjdir).replace(os.sep, '/')) + if dir: + return dir + return '.' + + def get_top_srcdir(self, file): + '''Returns a normalized top_srcdir for the given file: if + substs['top_srcdir'] is a relative path, it is relative to the + topobjdir. Adjust it to be relative to the file path.''' + top_srcdir = self.substs['top_srcdir'] + if posixpath.isabs(top_srcdir) or ntpath.isabs(top_srcdir): + return top_srcdir + return posixpath.normpath(posixpath.join(self.get_depth(file), top_srcdir)) + + def get_file_srcdir(self, file): + '''Returns the srcdir for the given file, where srcdir is in msys + format on windows, thus derived from top_srcdir. + ''' + dir = self.get_relative_srcdir(file) + top_srcdir = self.get_top_srcdir(file) + return posixpath.normpath(posixpath.join(top_srcdir, dir)) + + def get_depth(self, file): + '''Returns the DEPTH for the given file, that is, the path to the + object directory relative to the directory containing the given file. + Always uses / as a path separator. + ''' + return relpath(self.topobjdir, os.path.dirname(file)).replace(os.sep, '/') + + def get_input(self, file): + '''Returns the input file path in the source tree that can be used + to create the given config file or header. + ''' + assert(isinstance(file, basestring)) + return os.path.normpath(os.path.join(self.topsrcdir, "%s.in" % relpath(file, self.topobjdir))) + + def create_config_file(self, path): + '''Creates the given config file. A config file is generated by + taking the corresponding source file and replacing occurences of + "@VAR@" by the value corresponding to "VAR" in the substs dict. + + Additional substs are defined according to the file being treated: + "srcdir" for its the path to its source directory + "relativesrcdir" for its source directory relative to the top + "DEPTH" for the path to the top object directory + ''' + input = self.get_input(path) + pp = Preprocessor() + pp.context.update(self.substs) + pp.context.update(top_srcdir = self.get_top_srcdir(path)) + pp.context.update(srcdir = self.get_file_srcdir(path)) + pp.context.update(relativesrcdir = self.get_relative_srcdir(path)) + pp.context.update(DEPTH = self.get_depth(path)) + pp.do_filter('attemptSubstitution') + pp.setMarker(None) + with FileAvoidWrite(path) as pp.out: + pp.do_include(input) + + def create_config_header(self, path): + '''Creates the given config header. A config header is generated by + taking the corresponding source file and replacing some #define/#undef + occurences: + "#undef NAME" is turned into "#define NAME VALUE" + "#define NAME" is unchanged + "#define NAME ORIGINAL_VALUE" is turned into "#define NAME VALUE" + "#undef UNKNOWN_NAME" is turned into "/* #undef UNKNOWN_NAME */" + Whitespaces are preserved. + ''' + with open(self.get_input(path), 'rU') as input: + ensureParentDir(path) + output = FileAvoidWrite(path) + r = re.compile('^\s*#\s*(?P<cmd>[a-z]+)(?:\s+(?P<name>\S+)(?:\s+(?P<value>\S+))?)?', re.U) + for l in input: + m = r.match(l) + if m: + cmd = m.group('cmd') + name = m.group('name') + value = m.group('value') + if name: + if name in self.defines: + if cmd == 'define' and value: + l = l[:m.start('value')] + str(self.defines[name]) + l[m.end('value'):] + elif cmd == 'undef': + l = l[:m.start('cmd')] + 'define' + l[m.end('cmd'):m.end('name')] + ' ' + str(self.defines[name]) + l[m.end('name'):] + elif cmd == 'undef': + l = '/* ' + l[:m.end('name')] + ' */' + l[m.end('name'):] + + output.write(l) + output.close() +
rename from build/tests/unit-ConfigStatus.py rename to python/mozbuild/mozbuild/test/backend/test_configenvironment.py --- a/build/tests/unit-ConfigStatus.py +++ b/python/mozbuild/mozbuild/test/backend/test_configenvironment.py @@ -1,68 +1,36 @@ -from __future__ import with_statement +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + import os, posixpath from StringIO import StringIO import unittest from mozunit import main, MockedOpen -import ConfigStatus -from ConfigStatus import FileAvoidWrite + +import mozbuild.backend.configenvironment as ConfigStatus class ConfigEnvironment(ConfigStatus.ConfigEnvironment): - def __init__(self, **args): - ConfigStatus.ConfigEnvironment.__init__(self, **args) + def __init__(self, *args, **kwargs): + ConfigStatus.ConfigEnvironment.__init__(self, *args, **kwargs) # Be helpful to unit tests if not 'top_srcdir' in self.substs: if os.path.isabs(self.topsrcdir): self.substs['top_srcdir'] = self.topsrcdir.replace(os.sep, '/') else: - self.substs['top_srcdir'] = ConfigStatus.relpath(self.topsrcdir, self.topobjdir).replace(os.sep, '/') - -class TestFileAvoidWrite(unittest.TestCase): - def test_file_avoid_write(self): - '''Test the FileAvoidWrite class - ''' - with MockedOpen({'file': 'content'}): - # Overwriting an existing file replaces its content - with FileAvoidWrite('file') as file: - file.write('bazqux') - self.assertEqual(open('file', 'r').read(), 'bazqux') - - # Creating a new file (obviously) stores its content - with FileAvoidWrite('file2') as file: - file.write('content') - self.assertEqual(open('file2').read(), 'content') - - class MyMockedOpen(MockedOpen): - '''MockedOpen extension to raise an exception if something - attempts to write in an opened file. - ''' - def __call__(self, name, mode): - if 'w' in mode: - raise Exception, 'Unexpected open with write mode' - return MockedOpen.__call__(self, name, mode) - - with MyMockedOpen({'file': 'content'}): - # Validate that MyMockedOpen works as intended - file = FileAvoidWrite('file') - file.write('foobar') - self.assertRaises(Exception, file.close) - - # Check that no write actually happens when writing the - # same content as what already is in the file - with FileAvoidWrite('file') as file: - file.write('content') + self.substs['top_srcdir'] = os.path.relpath(self.topsrcdir, self.topobjdir).replace(os.sep, '/') class TestEnvironment(unittest.TestCase): def test_auto_substs(self): '''Test the automatically set values of ACDEFINES, ALLDEFINES and ALLSUBSTS. ''' - env = ConfigEnvironment( + env = ConfigEnvironment('.', '.', defines = [ ('foo', 'bar'), ('baz', 'qux 42'), ('abc', 'def'), ('extra', 'foobar') ], non_global_defines = ['extra', 'ignore'], substs = [ ('FOO', 'bar'), ('ABC', 'def'), ('bar', 'baz qux'), ('zzz', '"abc def"') ]) # non_global_defines should be filtered out in ACDEFINES and # ALLDEFINES. # Original order of the defines need to be respected in ACDEFINES @@ -81,17 +49,17 @@ zzz = "abc def"''') def test_config_file(self): '''Test the creation of config files. ''' with MockedOpen({'file.in': '''#ifdef foo @foo@ @bar@ '''}): - env = ConfigEnvironment(substs = [ ('foo', 'bar baz') ]) + env = ConfigEnvironment('.', '.', substs = [ ('foo', 'bar baz') ]) env.create_config_file('file') self.assertEqual(open('file', 'r').read(), '''#ifdef foo bar baz @bar@ ''') def test_config_header(self): '''Test the creation of config headers. @@ -108,17 +76,17 @@ bar baz # undef baz #ifdef foo # undef foo # define foo 42 # define foo 42 #endif '''}): - env = ConfigEnvironment(defines = [ ('foo', 'baz qux'), ('baz', 1) ]) + env = ConfigEnvironment('.', '.', defines = [ ('foo', 'baz qux'), ('baz', 1) ]) env.create_config_header('file') self.assertEqual(open('file','r').read(), ''' /* Comment */ #define foo #define foo baz qux #define foo baz qux #define bar #define bar 42
--- a/python/mozbuild/mozbuild/test/test_util.py +++ b/python/mozbuild/mozbuild/test/test_util.py @@ -1,21 +1,27 @@ # This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import unicode_literals import hashlib import unittest from mozfile.mozfile import NamedTemporaryFile -from mozunit import main +from mozunit import ( + main, + MockedOpen, +) -from mozbuild.util import hash_file +from mozbuild.util import ( + FileAvoidWrite, + hash_file, +) class TestHashing(unittest.TestCase): def test_hash_file_known_hash(self): """Ensure a known hash value is recreated.""" data = b'The quick brown fox jumps over the lazy cog' expected = 'de9f2c7fd25e1b3afad3e85a0bd17d9b100db4b3' @@ -39,10 +45,44 @@ class TestHashing(unittest.TestCase): temp.write(data) temp.flush() actual = hash_file(temp.name) self.assertEqual(actual, expected) +class TestFileAvoidWrite(unittest.TestCase): + def test_file_avoid_write(self): + with MockedOpen({'file': 'content'}): + # Overwriting an existing file replaces its content + with FileAvoidWrite('file') as file: + file.write('bazqux') + self.assertEqual(open('file', 'r').read(), 'bazqux') + + # Creating a new file (obviously) stores its content + with FileAvoidWrite('file2') as file: + file.write('content') + self.assertEqual(open('file2').read(), 'content') + + class MyMockedOpen(MockedOpen): + '''MockedOpen extension to raise an exception if something + attempts to write in an opened file. + ''' + def __call__(self, name, mode): + if 'w' in mode: + raise Exception, 'Unexpected open with write mode' + return MockedOpen.__call__(self, name, mode) + + with MyMockedOpen({'file': 'content'}): + # Validate that MyMockedOpen works as intended + file = FileAvoidWrite('file') + file.write('foobar') + self.assertRaises(Exception, file.close) + + # Check that no write actually happens when writing the + # same content as what already is in the file + with FileAvoidWrite('file') as file: + file.write('content') + + if __name__ == '__main__': main()
--- a/python/mozbuild/mozbuild/util.py +++ b/python/mozbuild/mozbuild/util.py @@ -3,17 +3,21 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. # This file contains miscellaneous utility functions that don't belong anywhere # in particular. from __future__ import unicode_literals import copy +import errno import hashlib +import os + +from StringIO import StringIO def hash_file(path): """Hashes a file specified by the path given and returns the hex digest.""" # If the hashing function changes, this may invalidate lots of cached data. # Don't change it lightly. h = hashlib.sha1() @@ -79,8 +83,57 @@ class DefaultOnReadDict(dict): return dict.__getitem__(self, k) class ReadOnlyDefaultDict(DefaultOnReadDict, ReadOnlyDict): """A read-only dictionary that supports default values on retrieval.""" def __init__(self, d, defaults=None, global_default=undefined): DefaultOnReadDict.__init__(self, d, defaults, global_default) + +def ensureParentDir(path): + """Ensures the directory parent to the given file exists.""" + d = os.path.dirname(path) + if d and not os.path.exists(path): + try: + os.makedirs(d) + except OSError, error: + if error.errno != errno.EEXIST: + raise + + +class FileAvoidWrite(StringIO): + """File-like object that buffers output and only writes if content changed. + + We create an instance from an existing filename. New content is written to + it. When we close the file object, if the content in the in-memory buffer + differs from what is on disk, then we write out the new content. Otherwise, + the original file is untouched. + """ + def __init__(self, filename): + StringIO.__init__(self) + self.filename = filename + + def close(self): + buf = self.getvalue() + StringIO.close(self) + try: + existing = open(self.filename, 'rU') + except IOError: + pass + else: + try: + if existing.read() == buf: + return + except IOError: + pass + finally: + existing.close() + + ensureParentDir(self.filename) + with open(self.filename, 'w') as file: + file.write(buf) + + def __enter__(self): + return self + def __exit__(self, type, value, traceback): + self.close() +