Update pymake snapshot to r293.
authorKyle Huey <khuey@kylehuey.com>
Thu, 21 Jul 2011 11:48:23 -0700
changeset 73179 ec7c3f978997e1134a4a2828eb80c0ef3a083ee2
parent 73178 aa5bfc851c357e5e059528dd2670ea79c37b74b1
child 73180 bf397db9ccfcf608831e50f944e13c08fd397e88
push idunknown
push userunknown
push dateunknown
milestone8.0a1
Update pymake snapshot to r293.
build/pymake/pymake/command.py
build/pymake/pymake/data.py
build/pymake/pymake/implicit.py
build/pymake/pymake/process.py
build/pymake/tests/mkdir-fail.mk
build/pymake/tests/runtests.py
--- a/build/pymake/pymake/command.py
+++ b/build/pymake/pymake/command.py
@@ -100,19 +100,23 @@ class _MakeContext(object):
             if self.restarts > 0:
                 _log.info("make.py[%i]: Restarting makefile parsing", self.makelevel)
 
             self.makefile = data.Makefile(restarts=self.restarts,
                                           make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
                                           makeflags=self.makeflags,
                                           makeoverrides=self.overrides,
                                           workdir=self.workdir,
-                                          context=self.context, env=self.env, makelevel=self.makelevel,
-                                          targets=self.targets, keepgoing=self.options.keepgoing,
-                                          silent=self.options.silent)
+                                          context=self.context,
+                                          env=self.env,
+                                          makelevel=self.makelevel,
+                                          targets=self.targets,
+                                          keepgoing=self.options.keepgoing,
+                                          silent=self.options.silent,
+                                          justprint=self.options.justprint)
 
             self.restarts += 1
 
             try:
                 self.ostmts.execute(self.makefile)
                 for f in self.options.makefiles:
                     self.makefile.include(f)
                 self.makefile.finishparsing()
@@ -184,16 +188,19 @@ def main(args, env, cwd, cb):
         op.add_option('-j', '--jobs', type="int",
                       dest="jobcount", default=1)
         op.add_option('-w', '--print-directory', action="store_true",
                       dest="printdir")
         op.add_option('--no-print-directory', action="store_false",
                       dest="printdir", default=True)
         op.add_option('-s', '--silent', action="store_true",
                       dest="silent", default=False)
+        op.add_option('-n', '--just-print', '--dry-run', '--recon',
+                      action="store_true",
+                      dest="justprint", default=False)
 
         options, arguments1 = op.parse_args(parsemakeflags(env))
         options, arguments2 = op.parse_args(args, values=options)
 
         op.destroy()
 
         arguments = arguments1 + arguments2
 
@@ -210,16 +217,19 @@ def main(args, env, cwd, cb):
 
         if options.printdir:
             shortflags.append('w')
 
         if options.silent:
             shortflags.append('s')
             options.printdir = False
 
+        if options.justprint:
+            shortflags.append('n')
+
         loglevel = logging.WARNING
         if options.verbose:
             loglevel = logging.DEBUG
             shortflags.append('d')
 
         logkwargs = {}
         if options.debuglog:
             logkwargs['filename'] = options.debuglog
--- a/build/pymake/pymake/data.py
+++ b/build/pymake/pymake/data.py
@@ -1207,31 +1207,32 @@ def getcommandsforrule(rule, target, mak
         setautomatic(v, '*', [stem])
 
     env = makefile.getsubenvironment(v)
 
     for c in rule.commands:
         cstring = c.resolvestr(makefile, v)
         for cline in splitcommand(cstring):
             cline, isHidden, isRecursive, ignoreErrors, isNative = findmodifiers(cline)
-            if isHidden or makefile.silent:
+            if (isHidden or makefile.silent) and not makefile.justprint:
                 echo = None
             else:
                 echo = "%s$ %s" % (c.loc, cline)
             if not isNative:
                 yield _CommandWrapper(cline, ignoreErrors=ignoreErrors, env=env, cwd=makefile.workdir, loc=c.loc, context=makefile.context,
-                                      echo=echo)
+                                      echo=echo, justprint=makefile.justprint)
             else:
                 f, s, e = v.get("PYCOMMANDPATH", True)
                 if e:
                     e = e.resolvestr(makefile, v, ["PYCOMMANDPATH"])
                 yield _NativeWrapper(cline, ignoreErrors=ignoreErrors,
                                      env=env, cwd=makefile.workdir,
                                      loc=c.loc, context=makefile.context,
-                                     echo=echo, pycommandpath=e)
+                                     echo=echo, justprint=makefile.justprint,
+                                     pycommandpath=e)
 
 class Rule(object):
     """
     A rule contains a list of prerequisites and a list of commands. It may also
     contain rule-specific variables. This rule may be associated with multiple targets.
     """
 
     def __init__(self, prereqs, doublecolon, loc, weakdeps):
@@ -1367,31 +1368,32 @@ class Makefile(object):
     """
     The top-level data structure for makefile execution. It holds Targets, implicit rules, and other
     state data.
     """
 
     def __init__(self, workdir=None, env=None, restarts=0, make=None,
                  makeflags='', makeoverrides='',
                  makelevel=0, context=None, targets=(), keepgoing=False,
-                 silent=False):
+                 silent=False, justprint=False):
         self.defaulttarget = None
 
         if env is None:
             env = os.environ
         self.env = env
 
         self.variables = Variables()
         self.variables.readfromenvironment(env)
 
         self.context = context
         self.exportedvars = {}
         self._targets = {}
         self.keepgoing = keepgoing
         self.silent = silent
+        self.justprint = justprint
         self._patternvariables = [] # of (pattern, variables)
         self.implicitrules = []
         self.parsingfinished = False
 
         self._patternvpaths = [] # of (pattern, [dir, ...])
 
         if workdir is None:
             workdir = os.getcwd()
--- a/build/pymake/pymake/implicit.py
+++ b/build/pymake/pymake/implicit.py
@@ -1,14 +1,14 @@
-"""
-Implicit variables; perhaps in the future this will also include some implicit
-rules, at least match-anything cancellation rules.
-"""
-
-variables = {
-    'MKDIR': '%pymake.builtins mkdir',
-    'RM': '%pymake.builtins rm -f',
-    'SLEEP': '%pymake.builtins sleep',
-    'TOUCH': '%pymake.builtins touch',
-    '.LIBPATTERNS': 'lib%.so lib%.a',
-    '.PYMAKE': '1',
-    }
-
+"""
+Implicit variables; perhaps in the future this will also include some implicit
+rules, at least match-anything cancellation rules.
+"""
+
+variables = {
+    'MKDIR': '%pymake.builtins mkdir',
+    'RM': '%pymake.builtins rm -f',
+    'SLEEP': '%pymake.builtins sleep',
+    'TOUCH': '%pymake.builtins touch',
+    '.LIBPATTERNS': 'lib%.so lib%.a',
+    '.PYMAKE': '1',
+    }
+
--- a/build/pymake/pymake/process.py
+++ b/build/pymake/pymake/process.py
@@ -9,43 +9,45 @@ import subprocess, shlex, re, logging, s
 # XXXkhuey Work around http://bugs.python.org/issue1731717
 subprocess._cleanup = lambda: None
 import command, util
 if sys.platform=='win32':
     import win32process
 
 _log = logging.getLogger('pymake.process')
 
-_blacklist = re.compile(r'[$><;*?[{~`|&]|\\\n')
+_escapednewlines = re.compile(r'\\\n')
+_blacklist = re.compile(r'[$><;*?[{~`|&]')
 def clinetoargv(cline):
     """
     If this command line can safely skip the shell, return an argv array.
     @returns argv, badchar
     """
 
-    m = _blacklist.search(cline)
+    str = _escapednewlines.sub('', cline)
+    m = _blacklist.search(str)
     if m is not None:
         return None, m.group(0)
 
-    args = shlex.split(cline, comments=True)
+    args = shlex.split(str, comments=True)
 
     if len(args) and args[0].find('=') != -1:
         return None, '='
 
     return args, None
 
 shellwords = (':', '.', 'break', 'cd', 'continue', 'exec', 'exit', 'export',
               'getopts', 'hash', 'pwd', 'readonly', 'return', 'shift', 
               'test', 'times', 'trap', 'umask', 'unset', 'alias',
               'set', 'bind', 'builtin', 'caller', 'command', 'declare',
               'echo', 'enable', 'help', 'let', 'local', 'logout', 
               'printf', 'read', 'shopt', 'source', 'type', 'typeset',
               'ulimit', 'unalias', 'set')
 
-def call(cline, env, cwd, loc, cb, context, echo):
+def call(cline, env, cwd, loc, cb, context, echo, justprint=False):
     #TODO: call this once up-front somewhere and save the result?
     shell, msys = util.checkmsyscompat()
 
     shellreason = None
     if msys and cline.startswith('/'):
         shellreason = "command starts with /"
     else:
         argv, badchar = clinetoargv(cline)
@@ -55,17 +57,18 @@ def call(cline, env, cwd, loc, cb, conte
             shellreason = "command starts with shell primitive '%s'" % (argv[0],)
 
     if shellreason is not None:
         _log.debug("%s: using shell: %s: '%s'", loc, shellreason, cline)
         if msys:
             if len(cline) > 3 and cline[1] == ':' and cline[2] == '/':
                 cline = '/' + cline[0] + cline[2:]
             cline = [shell, "-c", cline]
-        context.call(cline, shell=not msys, env=env, cwd=cwd, cb=cb, echo=echo)
+        context.call(cline, shell=not msys, env=env, cwd=cwd, cb=cb, echo=echo,
+                     justprint=justprint)
         return
 
     if not len(argv):
         cb(res=0)
         return
 
     if argv[0] == command.makepypath:
         command.main(argv[1:], env, cwd, cb)
@@ -76,22 +79,23 @@ def call(cline, env, cwd, loc, cb, conte
         command.main(argv[2:], env, cwd, cb)
         return
 
     if argv[0].find('/') != -1:
         executable = util.normaljoin(cwd, argv[0])
     else:
         executable = None
 
-    context.call(argv, executable=executable, shell=False, env=env, cwd=cwd, cb=cb, echo=echo)
+    context.call(argv, executable=executable, shell=False, env=env, cwd=cwd, cb=cb,
+                 echo=echo, justprint=justprint)
 
-def call_native(module, method, argv, env, cwd, loc, cb, context, echo,
+def call_native(module, method, argv, env, cwd, loc, cb, context, echo, justprint=False,
                 pycommandpath=None):
     context.call_native(module, method, argv, env=env, cwd=cwd, cb=cb,
-                        echo=echo, pycommandpath=pycommandpath)
+                        echo=echo, justprint=justprint, pycommandpath=pycommandpath)
 
 def statustoresult(status):
     """
     Convert the status returned from waitpid into a prettier numeric result.
     """
     sig = status & 0xFF
     if sig:
         return -sig
@@ -237,46 +241,42 @@ class ParallelContext(object):
         while len(self.pending) and len(self.running) < self.jcount:
             cb, args, kwargs = self.pending.pop(0)
             cb(*args, **kwargs)
 
     def defer(self, cb, *args, **kwargs):
         assert self.jcount > 1 or not len(self.pending), "Serial execution error defering %r %r %r: currently pending %r" % (cb, args, kwargs, self.pending)
         self.pending.append((cb, args, kwargs))
 
-    def _docall(self, argv, executable, shell, env, cwd, cb, echo):
+    def _docall_generic(self, pool, job, cb, echo, justprint):
         if echo is not None:
             print echo
-        job = PopenJob(argv, executable=executable, shell=shell, env=env, cwd=cwd)
-        self.threadpool.apply_async(job_runner, args=(job,), callback=job.get_callback(ParallelContext._condition))
+        processcb = job.get_callback(ParallelContext._condition)
+        if justprint:
+            processcb(0)
+        else:
+            pool.apply_async(job_runner, args=(job,), callback=processcb)
         self.running.append((job, cb))
 
-    def _docallnative(self, module, method, argv, env, cwd, cb, echo,
-                      pycommandpath=None):
-        if echo is not None:
-            print echo
-        job = PythonJob(module, method, argv, env, cwd, pycommandpath)
-        self.processpool.apply_async(job_runner, args=(job,), callback=job.get_callback(ParallelContext._condition))
-        self.running.append((job, cb))
-
-    def call(self, argv, shell, env, cwd, cb, echo, executable=None):
+    def call(self, argv, shell, env, cwd, cb, echo, justprint=False, executable=None):
         """
         Asynchronously call the process
         """
 
-        self.defer(self._docall, argv, executable, shell, env, cwd, cb, echo)
+        job = PopenJob(argv, executable=executable, shell=shell, env=env, cwd=cwd)
+        self.defer(self._docall_generic, self.threadpool, job, cb, echo, justprint)
 
     def call_native(self, module, method, argv, env, cwd, cb,
-                    echo, pycommandpath=None):
+                    echo, justprint=False, pycommandpath=None):
         """
         Asynchronously call the native function
         """
 
-        self.defer(self._docallnative, module, method, argv, env, cwd, cb,
-                   echo, pycommandpath)
+        job = PythonJob(module, method, argv, env, cwd, pycommandpath)
+        self.defer(self._docall_generic, self.processpool, job, cb, echo, justprint)
 
     @staticmethod
     def _waitany(condition):
         def _checkdone():
             jobs = []
             for c in ParallelContext._allcontexts:
                 for i in xrange(0, len(c.running)):
                     if c.running[i][0].done:
--- a/build/pymake/tests/mkdir-fail.mk
+++ b/build/pymake/tests/mkdir-fail.mk
@@ -1,7 +1,7 @@
-#T returncode: 1
+#T returncode: 2
 all:
 	mkdir newdir/subdir
 	test ! -d newdir/subdir
 	test ! -d newdir
 	rm -r newdir
 	@echo TEST-PASS
--- a/build/pymake/tests/runtests.py
+++ b/build/pymake/tests/runtests.py
@@ -73,22 +73,25 @@ def runTest(makefile, make, logfile, opt
     if retcode != options['returncode']:
         return False, "FAIL (returncode=%i)" % retcode
         
     logfd = open(logfile)
     stdout = logfd.read()
     logfd.close()
 
     if stdout.find('TEST-FAIL') != -1:
+        print stdout
         return False, "FAIL (TEST-FAIL printed)"
 
     if options['grepfor'] and stdout.find(options['grepfor']) == -1:
-            return False, "FAIL (%s not in output)" % options['grepfor']
+        print stdout
+        return False, "FAIL (%s not in output)" % options['grepfor']
 
     if options['returncode'] == 0 and stdout.find('TEST-PASS') == -1:
+        print stdout
         return False, 'FAIL (No TEST-PASS printed)'
 
     if options['returncode'] != 0:
         return True, 'PASS (retcode=%s)' % retcode
 
     return True, 'PASS'
 
 print "%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:")
@@ -118,16 +121,17 @@ for makefile in makefiles:
 
     gmakeoptions = ParentDict(options)
     pymakeoptions = ParentDict(options)
 
     dmap = {None: options, 'gmake ': gmakeoptions, 'pymake ': pymakeoptions}
 
     mdata = open(makefile)
     for line in mdata:
+        line = line.strip()
         m = tre.search(line)
         if m is None:
             break
 
         make, key, data = m.group(1, 2, 3)
         d = dmap[make]
         if data is not None:
             data = eval(data)