Update pymake
authorBenjamin Smedberg <benjamin@smedbergs.us>
Thu, 02 Apr 2009 12:26:17 -0400
changeset 26868 98b6e7826dfc342b0f20e78c6cb95d090d5f753b
parent 26867 8e464707cf5bdef08a71fe3b7de6758f62058928
child 26869 5115ae412a86007105717cfe94b10438af9a0c9e
push id1
push userroot
push dateTue, 26 Apr 2011 22:38:44 +0000
treeherdermozilla-esr52@a95d42642281 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
milestone1.9.2a1pre
Update pymake
build/pymake/.hg_archival.txt
build/pymake/pymake/command.py
build/pymake/pymake/data.py
build/pymake/pymake/functions.py
build/pymake/pymake/parserdata.py
build/pymake/tests/override-propagate.mk
build/pymake/tests/simple-makeflags.mk
--- a/build/pymake/.hg_archival.txt
+++ b/build/pymake/.hg_archival.txt
@@ -1,2 +1,2 @@
 repo: f5ab154deef2ffa97f1b2139589ae4a1962090a4
-node: ab32ac2a4e6842787fef44f43101c03ff515a3a3
+node: 162a3b79a8b0799a9fb3c2aed909a3d09f6f1d39
--- a/build/pymake/pymake/command.py
+++ b/build/pymake/pymake/command.py
@@ -10,17 +10,17 @@ import os, subprocess, sys, logging, tim
 from optparse import OptionParser
 import data, parserdata, process, util
 
 # TODO: If this ever goes from relocatable package to system-installed, this may need to be
 # a configured-in path.
 
 makepypath = os.path.normpath(os.path.join(os.path.dirname(__file__), '../make.py'))
 
-_simpleopts = re.compile(r'^[a-zA-Z]+\s')
+_simpleopts = re.compile(r'^[a-zA-Z]+(\s|$)')
 def parsemakeflags(env):
     """
     Parse MAKEFLAGS from the environment into a sequence of command-line arguments.
     """
 
     makeflags = env.get('MAKEFLAGS', '')
     makeflags = makeflags.strip()
 
@@ -68,47 +68,50 @@ FITNESS FOR A PARTICULAR PURPOSE AND NON
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 DEALINGS IN THE SOFTWARE."""
 
 _log = logging.getLogger('pymake.execution')
 
 class _MakeContext(object):
-    def __init__(self, makeflags, makelevel, workdir, context, env, targets, options, overrides, cb):
+    def __init__(self, makeflags, makelevel, workdir, context, env, targets, options, ostmts, overrides, cb):
         self.makeflags = makeflags
         self.makelevel = makelevel
 
         self.workdir = workdir
         self.context = context
         self.env = env
         self.targets = targets
         self.options = options
+        self.ostmts = ostmts
         self.overrides = overrides
         self.cb = cb
 
         self.restarts = 0
 
         self.remakecb(True)
 
     def remakecb(self, remade):
         if remade:
             if self.restarts > 0:
                 _log.info("make.py[%i]: Restarting makefile parsing", self.makelevel)
 
             self.makefile = data.Makefile(restarts=self.restarts,
                                           make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
-                                          makeflags=self.makeflags, workdir=self.workdir,
+                                          makeflags=self.makeflags,
+                                          makeoverrides=self.overrides,
+                                          workdir=self.workdir,
                                           context=self.context, env=self.env, makelevel=self.makelevel,
                                           targets=self.targets, keepgoing=self.options.keepgoing)
 
             self.restarts += 1
 
             try:
-                self.overrides.execute(self.makefile)
+                self.ostmts.execute(self.makefile)
                 for f in self.options.makefiles:
                     self.makefile.include(f)
                 self.makefile.finishparsing()
                 self.makefile.remakemakefiles(self.remakecb)
             except util.MakeError, e:
                 print e
                 self.context.defer(self.cb, 2)
 
@@ -190,36 +193,42 @@ def main(args, env, cwd, cb):
             _version()
             cb(0)
             return
 
         shortflags = []
         longflags = []
 
         if options.keepgoing:
-            shortflags.append('k');
+            shortflags.append('k')
+
+        if options.printdir:
+            shortflags.append('w')
 
         loglevel = logging.WARNING
         if options.verbose:
             loglevel = logging.DEBUG
             shortflags.append('d')
 
         logkwargs = {}
         if options.debuglog:
             logkwargs['filename'] = options.debuglog
             longflags.append('--debug-log=%s' % options.debuglog)
 
         if options.directory is None:
             workdir = cwd
         else:
             workdir = os.path.join(cwd, options.directory)
 
-        longflags.append('-j%i' % (options.jobcount,))
+        if options.jobcount != 1:
+            longflags.append('-j%i' % (options.jobcount,))
 
-        makeflags = ''.join(shortflags) + ' ' + ' '.join(longflags)
+        makeflags = ''.join(shortflags)
+        if len(longflags):
+            makeflags += ' ' + ' '.join(longflags)
 
         logging.basicConfig(level=loglevel, **logkwargs)
 
         context = process.getcontext(options.jobcount)
 
         if options.printdir:
             print "make.py[%i]: Entering directory '%s'" % (makelevel, workdir)
             sys.stdout.flush()
@@ -227,18 +236,18 @@ def main(args, env, cwd, cb):
         if len(options.makefiles) == 0:
             if os.path.exists(os.path.join(workdir, 'Makefile')):
                 options.makefiles.append('Makefile')
             else:
                 print "No makefile found"
                 cb(2)
                 return
 
-        overrides, targets = parserdata.parsecommandlineargs(arguments)
+        ostmts, targets, overrides = parserdata.parsecommandlineargs(arguments)
 
-        _MakeContext(makeflags, makelevel, workdir, context, env, targets, options, overrides, cb)
+        _MakeContext(makeflags, makelevel, workdir, context, env, targets, options, ostmts, overrides, cb)
     except (util.MakeError), e:
         print e
         if options.printdir:
             print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
         sys.stdout.flush()
         cb(2)
         return
--- a/build/pymake/pymake/data.py
+++ b/build/pymake/pymake/data.py
@@ -1293,29 +1293,30 @@ class _RemakeContext(object):
             self.cb(remade=False)
 
 class Makefile(object):
     """
     The top-level data structure for makefile execution. It holds Targets, implicit rules, and other
     state data.
     """
 
-    def __init__(self, workdir=None, env=None, restarts=0, make=None, makeflags=None, makelevel=0, context=None, targets=(), keepgoing=False):
+    def __init__(self, workdir=None, env=None, restarts=0, make=None,
+                 makeflags='', makeoverrides='',
+                 makelevel=0, context=None, targets=(), keepgoing=False):
         self.defaulttarget = None
 
         if env is None:
             env = os.environ
         self.env = env
 
         self.variables = Variables()
         self.variables.readfromenvironment(env)
 
         self.context = context
         self.exportedvars = {}
-        self.overrides = []
         self._targets = {}
         self.keepgoing = keepgoing
         self._patternvariables = [] # of (pattern, variables)
         self.implicitrules = []
         self.parsingfinished = False
 
         self._patternvpaths = [] # of (pattern, [dir, ...])
 
@@ -1331,29 +1332,39 @@ class Makefile(object):
 
         self.variables.set('MAKE_RESTARTS', Variables.FLAVOR_SIMPLE,
                            Variables.SOURCE_AUTOMATIC, restarts > 0 and str(restarts) or '')
 
         if make is not None:
             self.variables.set('MAKE', Variables.FLAVOR_SIMPLE,
                                Variables.SOURCE_MAKEFILE, make)
 
-        if makeflags is not None:
-            self.variables.set('MAKEFLAGS', Variables.FLAVOR_SIMPLE,
-                               Variables.SOURCE_MAKEFILE, makeflags)
+        if makeoverrides != '':
+            self.variables.set('-*-command-variables-*-', Variables.FLAVOR_SIMPLE,
+                               Variables.SOURCE_AUTOMATIC, makeoverrides)
+            makeflags += ' -- $(MAKEOVERRIDES)'
+
+        self.variables.set('MAKEOVERRIDES', Variables.FLAVOR_RECURSIVE,
+                           Variables.SOURCE_ENVIRONMENT,
+                           '${-*-command-variables-*-}')
+
+        self.variables.set('MAKEFLAGS', Variables.FLAVOR_RECURSIVE,
+                           Variables.SOURCE_MAKEFILE, makeflags)
+        self.exportedvars['MAKEFLAGS'] = True
 
         self.makelevel = makelevel
         self.variables.set('MAKELEVEL', Variables.FLAVOR_SIMPLE,
                            Variables.SOURCE_MAKEFILE, str(makelevel))
 
         self.variables.set('MAKECMDGOALS', Variables.FLAVOR_SIMPLE,
                            Variables.SOURCE_AUTOMATIC, ' '.join(targets))
 
         for vname, val in builtins.variables.iteritems():
-            self.variables.set(vname, Variables.FLAVOR_SIMPLE,
+            self.variables.set(vname,
+                               Variables.FLAVOR_SIMPLE,
                                Variables.SOURCE_IMPLICIT, val)
 
     def foundtarget(self, t):
         """
         Inform the makefile of a target which is a candidate for being the default target,
         if there isn't already a default target.
         """
         if self.defaulttarget is None:
@@ -1478,38 +1489,25 @@ class Makefile(object):
             t.explicit = True
             t.resolvevpath(self)
             oldmtime = t.mtime
 
             mlist.append((t, oldmtime))
 
         _RemakeContext(self, [self.gettarget(f) for f in self.included], mlist, cb)
 
-    flagescape = re.compile(r'([\s\\])')
-
     def getsubenvironment(self, variables):
         env = dict(self.env)
         for vname, v in self.exportedvars.iteritems():
             if v:
                 flavor, source, val = variables.get(vname)
                 if val is None:
                     strval = ''
                 else:
                     strval = val.resolvestr(self, variables, [vname])
                 env[vname] = strval
             else:
                 env.pop(vname, None)
 
         makeflags = ''
 
-        flavor, source, val = variables.get('MAKEFLAGS')
-        if val is not None:
-            flagsval = val.resolvestr(self, variables, ['MAKEFLAGS'])
-            if flagsval != '':
-                makeflags = flagsval
-
-        makeflags += ' -- '
-        makeflags += ' '.join((self.flagescape.sub(r'\\\1', o) for o in self.overrides))
-
-        env['MAKEFLAGS'] = makeflags
-
         env['MAKELEVEL'] = str(self.makelevel + 1)
         return env
--- a/build/pymake/pymake/functions.py
+++ b/build/pymake/pymake/functions.py
@@ -569,16 +569,18 @@ class OriginFunction(Function):
         elif source == data.Variables.SOURCE_MAKEFILE:
             r = 'file'
         elif source == data.Variables.SOURCE_ENVIRONMENT:
             r = 'environment'
         elif source == data.Variables.SOURCE_COMMANDLINE:
             r = 'command line'
         elif source == data.Variables.SOURCE_AUTOMATIC:
             r = 'automatic'
+        elif source == data.Variables.SOURCE_IMPLICIT:
+            r = 'default'
 
         fd.write(r)
 
 class FlavorFunction(Function):
     name = 'flavor'
     minargs = 1
     maxargs = 1
 
--- a/build/pymake/pymake/parserdata.py
+++ b/build/pymake/pymake/parserdata.py
@@ -61,64 +61,55 @@ def _expandwildcards(makefile, tlist):
     for t in tlist:
         if not hasglob(t):
             yield t
         else:
             l = glob(makefile.workdir, t)
             for r in l:
                 yield r
 
+_flagescape = re.compile(r'([\s\\])')
+
 def parsecommandlineargs(args):
     """
     Given a set of arguments from a command-line invocation of make,
-    parse out the variable definitions and return (stmts, arglist)
+    parse out the variable definitions and return (stmts, arglist, overridestr)
     """
 
+    overrides = []
     stmts = StatementList()
     r = []
     for i in xrange(0, len(args)):
         a = args[i]
 
         vname, t, val = util.strpartition(a, ':=')
         if t == '':
             vname, t, val = util.strpartition(a, '=')
         if t != '':
-            stmts.append(Override(a))
+            overrides.append(_flagescape.sub(r'\\\1', a))
 
             vname = vname.strip()
             vnameexp = data.Expansion.fromstring(vname, "Command-line argument")
 
             stmts.append(SetVariable(vnameexp, token=t,
                                      value=val, valueloc=Location('<command-line>', i, len(vname) + len(t)),
                                      targetexp=None, source=data.Variables.SOURCE_COMMANDLINE))
         else:
             r.append(a)
 
-    return stmts, r
+    return stmts, r, ' '.join(overrides)
 
 class Statement(object):
     """
     A statement is an abstract object representing a single "chunk" of makefile syntax. Subclasses
     must implement the following method:
 
     def execute(self, makefile, context)
     """
 
-class Override(Statement):
-    __slots__ = ('s',)
-
-    def __init__(self, s):
-        self.s = s
-
-    def execute(self, makefile, context):
-        makefile.overrides.append(self.s)
-
-    def dump(self, fd, indent):
-        print >>fd, "%sOverride: %s" % (indent, self.s)
-
 class DummyRule(object):
     __slots__ = ()
 
     def addcommand(self, r):
         pass
 
 class Rule(Statement):
     __slots__ = ('targetexp', 'depexp', 'doublecolon')
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/override-propagate.mk
@@ -0,0 +1,32 @@
+#T commandline: ['-w', 'OVAR=oval']
+
+OVAR=mval
+
+all: vartest run-override
+	$(MAKE) -f $(TESTPATH)/override-propagate.mk vartest
+	@echo TEST-PASS
+
+SORTED_CLINE := $(sort OVAR=oval TESTPATH=$(TESTPATH) NATIVE_TESTPATH=$(NATIVE_TESTPATH))
+
+vartest:
+	@echo MAKELEVEL: '$(MAKELEVEL)'
+	test '$(value MAKEFLAGS)' = 'w -- $$(MAKEOVERRIDES)'
+	test '$(origin MAKEFLAGS)' = 'file'
+	test '$(value MAKEOVERRIDES)' = '$${-*-command-variables-*-}'
+	test "$(sort $(MAKEOVERRIDES))" = "$(SORTED_CLINE)"
+	test '$(origin MAKEOVERRIDES)' = 'environment'
+	test '$(origin -*-command-variables-*-)' = 'automatic'
+	test "$(origin OVAR)" = "command line"
+	test "$(OVAR)" = "oval"
+
+run-override: MAKEOVERRIDES=
+run-override:
+	test "$(OVAR)" = "oval"
+	$(MAKE) -f $(TESTPATH)/override-propagate.mk otest
+
+otest:
+	test '$(value MAKEFLAGS)' = 'w'
+	test '$(value MAKEOVERRIDES)' = '$${-*-command-variables-*-}'
+	test '$(MAKEOVERRIDES)' = ''
+	test '$(origin -*-command-variables-*-)' = 'undefined'
+	test "$(OVAR)" = "mval"
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/simple-makeflags.mk
@@ -0,0 +1,10 @@
+# There once was a time when MAKEFLAGS=w without any following spaces would
+# cause us to treat w as a target, not a flag. Silly!
+
+MAKEFLAGS=w
+
+all:
+	$(MAKE) -f $(TESTPATH)/simple-makeflags.mk subt
+	@echo TEST-PASS
+
+subt: