Update pymake for better memory usage and faster execution.
authorBenjamin Smedberg <benjamin@smedbergs.us>
Wed, 01 Apr 2009 16:13:55 -0400
changeset 26835 e38c2273bbac33b49574bde4c7e68f17df0375d3
parent 26834 7348ed185ae2ee8233d2b955327bb5d9dc23a3ec
child 26836 35879b7a23f957ee34b89ce45f5baf205aea95c9
push id6250
push userbsmedberg@mozilla.com
push dateWed, 01 Apr 2009 20:14:17 +0000
treeherdermozilla-central@e38c2273bbac [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
milestone1.9.2a1pre
Update pymake for better memory usage and faster execution.
build/pymake/.hg_archival.txt
build/pymake/make.py
build/pymake/pymake/command.py
build/pymake/pymake/data.py
build/pymake/pymake/parser.py
build/pymake/pymake/parserdata.py
build/pymake/pymake/util.py
build/pymake/tests/datatests.py
--- a/build/pymake/.hg_archival.txt
+++ b/build/pymake/.hg_archival.txt
@@ -1,2 +1,2 @@
 repo: f5ab154deef2ffa97f1b2139589ae4a1962090a4
-node: 50d8e87e8af3fb8147c33f169a8d5abe8e06ffa4
+node: ab32ac2a4e6842787fef44f43101c03ff515a3a3
--- a/build/pymake/make.py
+++ b/build/pymake/make.py
@@ -4,11 +4,14 @@
 make.py
 
 A drop-in or mostly drop-in replacement for GNU make.
 """
 
 import sys, os
 import pymake.command, pymake.process
 
+import gc
+gc.disable()
+
 pymake.command.main(sys.argv[1:], os.environ, os.getcwd(), cb=sys.exit)
 pymake.process.ParallelContext.spin()
 assert False, "Not reached"
--- a/build/pymake/pymake/command.py
+++ b/build/pymake/pymake/command.py
@@ -67,16 +67,89 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO TH
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 DEALINGS IN THE SOFTWARE."""
 
 _log = logging.getLogger('pymake.execution')
 
+class _MakeContext(object):
+    def __init__(self, makeflags, makelevel, workdir, context, env, targets, options, overrides, cb):
+        self.makeflags = makeflags
+        self.makelevel = makelevel
+
+        self.workdir = workdir
+        self.context = context
+        self.env = env
+        self.targets = targets
+        self.options = options
+        self.overrides = overrides
+        self.cb = cb
+
+        self.restarts = 0
+
+        self.remakecb(True)
+
+    def remakecb(self, remade):
+        if remade:
+            if self.restarts > 0:
+                _log.info("make.py[%i]: Restarting makefile parsing", self.makelevel)
+
+            self.makefile = data.Makefile(restarts=self.restarts,
+                                          make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
+                                          makeflags=self.makeflags, workdir=self.workdir,
+                                          context=self.context, env=self.env, makelevel=self.makelevel,
+                                          targets=self.targets, keepgoing=self.options.keepgoing)
+
+            self.restarts += 1
+
+            try:
+                self.overrides.execute(self.makefile)
+                for f in self.options.makefiles:
+                    self.makefile.include(f)
+                self.makefile.finishparsing()
+                self.makefile.remakemakefiles(self.remakecb)
+            except util.MakeError, e:
+                print e
+                self.context.defer(self.cb, 2)
+
+            return
+
+        if len(self.targets) == 0:
+            if self.makefile.defaulttarget is None:
+                print "No target specified and no default target found."
+                self.context.defer(self.cb, 2)
+                return
+
+            _log.info("Making default target %s", self.makefile.defaulttarget)
+            self.realtargets = [self.makefile.defaulttarget]
+            self.tstack = ['<default-target>']
+        else:
+            self.realtargets = self.targets
+            self.tstack = ['<command-line>']
+
+        self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, cb=self.makecb)
+
+    def makecb(self, error, didanything):
+        assert error in (True, False)
+
+        if error:
+            self.context.defer(self.cb, 2)
+            return
+
+        if not len(self.realtargets):
+            if self.options.printdir:
+                print "make.py[%i]: Leaving directory '%s'" % (self.makelevel, self.workdir)
+            sys.stdout.flush()
+
+            self.context.defer(self.cb, 0)
+        else:
+            self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, self.makecb)
+
 def main(args, env, cwd, cb):
     """
     Start a single makefile execution, given a command line, working directory, and environment.
 
     @param cb a callback to notify with an exit code when make execution is finished.
     """
 
     try:
@@ -96,22 +169,26 @@ def main(args, env, cwd, cb):
         op.add_option('--debug-log',
                       dest="debuglog", default=None)
         op.add_option('-C', '--directory',
                       dest="directory", default=None)
         op.add_option('-v', '--version', action="store_true",
                       dest="printversion", default=False)
         op.add_option('-j', '--jobs', type="int",
                       dest="jobcount", default=1)
+        op.add_option('-w', '--print-directory', action="store_true",
+                      dest="printdir")
         op.add_option('--no-print-directory', action="store_false",
                       dest="printdir", default=True)
 
         options, arguments1 = op.parse_args(parsemakeflags(env))
         options, arguments2 = op.parse_args(args, values=options)
 
+        op.destroy()
+
         arguments = arguments1 + arguments2
 
         if options.printversion:
             _version()
             cb(0)
             return
 
         shortflags = []
@@ -152,78 +229,16 @@ def main(args, env, cwd, cb):
                 options.makefiles.append('Makefile')
             else:
                 print "No makefile found"
                 cb(2)
                 return
 
         overrides, targets = parserdata.parsecommandlineargs(arguments)
 
-        def makecb(error, didanything, makefile, realtargets, tstack, i):
-            assert error in (True, False)
-
-            if error:
-                context.defer(cb, 2)
-                return
-
-            if i == len(realtargets):
-                if options.printdir:
-                    print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
-                sys.stdout.flush()
-
-                context.defer(cb, 0)
-            else:
-                deferredmake = process.makedeferrable(makecb, makefile=makefile,
-                                                      realtargets=realtargets, tstack=tstack, i=i+1)
-
-                makefile.gettarget(realtargets[i]).make(makefile, tstack, cb=deferredmake)
-                                                                                  
-
-        def remakecb(remade, restarts, makefile):
-            if remade:
-                if restarts > 0:
-                    _log.info("make.py[%i]: Restarting makefile parsing", makelevel)
-                makefile = data.Makefile(restarts=restarts, make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
-                                         makeflags=makeflags, makelevel=makelevel, workdir=workdir,
-                                         context=context, env=env,
-                                         targets=targets,
-                                         keepgoing=options.keepgoing)
-
-                try:
-                    overrides.execute(makefile)
-                    for f in options.makefiles:
-                        makefile.include(f)
-                    makefile.finishparsing()
-                    makefile.remakemakefiles(process.makedeferrable(remakecb, restarts=restarts + 1, makefile=makefile))
-
-                except util.MakeError, e:
-                    print e
-                    context.defer(cb, 2)
-                    return
-
-                return
-
-            if len(targets) == 0:
-                if makefile.defaulttarget is None:
-                    print "No target specified and no default target found."
-                    context.defer(cb, 2)
-                    return
-
-                _log.info("Making default target %s", makefile.defaulttarget)
-                realtargets = [makefile.defaulttarget]
-                tstack = ['<default-target>']
-            else:
-                realtargets = targets
-                tstack = ['<command-line>']
-
-            deferredmake = process.makedeferrable(makecb, makefile=makefile,
-                                                  realtargets=realtargets, tstack=tstack, i=1)
-            makefile.gettarget(realtargets[0]).make(makefile, tstack, cb=deferredmake)
-
-        context.defer(remakecb, True, 0, None)
-
+        _MakeContext(makeflags, makelevel, workdir, context, env, targets, options, overrides, cb)
     except (util.MakeError), e:
         print e
         if options.printdir:
             print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
         sys.stdout.flush()
         cb(2)
         return
--- a/build/pymake/pymake/data.py
+++ b/build/pymake/pymake/data.py
@@ -1263,16 +1263,40 @@ class PatternRule(object):
                 else:
                     stem = p.match(file)
                     if stem is not None:
                         yield PatternRuleInstance(self, dir, stem, False)
 
     def prerequisitesforstem(self, dir, stem):
         return [p.resolve(dir, stem) for p in self.prerequisites]
 
+class _RemakeContext(object):
+    def __init__(self, makefile, remakelist, mtimelist, cb):
+        self.makefile = makefile
+        self.remakelist = remakelist
+        self.mtimelist = mtimelist # list of (target, mtime)
+        self.cb = cb
+
+        self.remakecb(error=False, didanything=False)
+
+    def remakecb(self, error, didanything):
+        assert error in (True, False)
+
+        if error:
+            print "Error remaking makefiles (ignored)"
+
+        if len(self.remakelist):
+            self.remakelist.pop(0).make(self.makefile, [], avoidremakeloop=True, cb=self.remakecb)
+        else:
+            for t, oldmtime in self.mtimelist:
+                if t.mtime != oldmtime:
+                    self.cb(remade=True)
+                    return
+            self.cb(remade=False)
+
 class Makefile(object):
     """
     The top-level data structure for makefile execution. It holds Targets, implicit rules, and other
     state data.
     """
 
     def __init__(self, workdir=None, env=None, restarts=0, make=None, makeflags=None, makelevel=0, context=None, targets=(), keepgoing=False):
         self.defaulttarget = None
@@ -1443,63 +1467,26 @@ class Makefile(object):
         vp = list(self._vpath)
         for p, dirs in self._patternvpaths:
             if p.match(target):
                 vp.extend(dirs)
 
         return withoutdups(vp)
 
     def remakemakefiles(self, cb):
-        reparse = False
-
-        serial = self.context.jcount == 1
-
-        def remakedone():
-            for t, oldmtime in mlist:
-                if t.mtime != oldmtime:
-                    cb(remade=True)
-                    return
-            cb(remade=False)
-
         mlist = []
         for f in self.included:
             t = self.gettarget(f)
             t.explicit = True
             t.resolvevpath(self)
             oldmtime = t.mtime
 
             mlist.append((t, oldmtime))
 
-        if serial:
-            remakelist = [self.gettarget(f) for f in self.included]
-            def remakecb(error, didanything):
-                assert error in (True, False)
-                if error:
-                    print "Error remaking makefiles (ignored)"
-
-                if len(remakelist):
-                    t = remakelist.pop(0)
-                    t.make(self, [], avoidremakeloop=True, cb=remakecb)
-                else:
-                    remakedone()
-
-            remakelist.pop(0).make(self, [], avoidremakeloop=True, cb=remakecb)
-        else:
-            o = util.makeobject(('remakesremaining',), remakesremaining=len(self.included))
-            def remakecb(error, didanything):
-                assert error in (True, False)
-                if error:
-                    print "Error remaking makefiles (ignored)"
-
-                o.remakesremaining -= 1
-                if o.remakesremaining == 0:
-                    remakedone()
-
-            for t, mtime in mlist:
-                t.make(self, [], avoidremakeloop=True, cb=remakecb)
+        _RemakeContext(self, [self.gettarget(f) for f in self.included], mlist, cb)
 
     flagescape = re.compile(r'([\s\\])')
 
     def getsubenvironment(self, variables):
         env = dict(self.env)
         for vname, v in self.exportedvars.iteritems():
             if v:
                 flavor, source, val = variables.get(vname)
--- a/build/pymake/pymake/parser.py
+++ b/build/pymake/pymake/parser.py
@@ -446,42 +446,41 @@ def ifndef(d, offset):
 
 _conditiontokens = tuple(_conditionkeywords.iterkeys())
 _directivestokenlist = TokenList.get(_conditiontokens + \
     ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'vpath', 'export', 'unexport'))
 _conditionkeywordstokenlist = TokenList.get(_conditiontokens)
 
 _varsettokens = (':=', '+=', '?=', '=')
 
-_parsecache = {} # realpath -> (mtime, Statements)
+def _parsefile(pathname):
+    fd = open(pathname, "rU")
+    stmts = parsestream(fd, pathname)
+    stmts.mtime = os.fstat(fd.fileno()).st_mtime
+    fd.close()
+    return stmts
+
+def _checktime(path, stmts):
+    mtime = os.path.getmtime(path)
+    if mtime != stmts.mtime:
+        _log.debug("Re-parsing makefile '%s': mtimes differ", path)
+        return False
+
+    return True
+
+_parsecache = util.MostUsedCache(15, _parsefile, _checktime)
 
 def parsefile(pathname):
     """
     Parse a filename into a parserdata.StatementList. A cache is used to avoid re-parsing
     makefiles that have already been parsed and have not changed.
     """
 
     pathname = os.path.realpath(pathname)
-
-    mtime = os.path.getmtime(pathname)
-
-    if pathname in _parsecache:
-        oldmtime, stmts = _parsecache[pathname]
-
-        if mtime == oldmtime:
-            _log.debug("Using '%s' from the parser cache.", pathname)
-            return stmts
-
-        _log.debug("Not using '%s' from the parser cache, mtimes don't match: was %s, now %s", pathname, oldmtime, mtime)
-
-    fd = open(pathname, "rU")
-    stmts = parsestream(fd, pathname)
-    fd.close()
-    _parsecache[pathname] = mtime, stmts
-    return stmts
+    return _parsecache.get(pathname)
 
 def parsestream(fd, filename):
     """
     Parse a stream of makefile into a parserdata.StatementList. To parse a file system file, use
     parsefile instead of this method.
 
     @param fd A file-like object containing the makefile data.
     """
--- a/build/pymake/pymake/parserdata.py
+++ b/build/pymake/pymake/parserdata.py
@@ -465,26 +465,29 @@ class EmptyDirective(Statement):
     def execute(self, makefile, context):
         v = self.exp.resolvestr(makefile, makefile.variables)
         if v.strip() != '':
             raise data.DataError("Line expands to non-empty value", self.exp.loc)
 
     def dump(self, fd, indent):
         print >>fd, "%sEmptyDirective: %s" % (indent, self.exp)
 
+class _EvalContext(object):
+    __slots__ = ('currule',)
+
 class StatementList(list):
-    __slots__ = ()
+    __slots__ = ('mtime',)
 
     def append(self, statement):
         assert isinstance(statement, Statement)
         list.append(self, statement)
 
     def execute(self, makefile, context=None):
         if context is None:
-            context = util.makeobject('currule')
+            context = _EvalContext()
 
         for s in self:
             s.execute(makefile, context)
 
     def dump(self, fd, indent):
         for s in self:
             s.dump(fd, indent)
 
--- a/build/pymake/pymake/util.py
+++ b/build/pymake/pymake/util.py
@@ -1,19 +1,10 @@
 import os
 
-def makeobject(proplist, **kwargs):
-    class P(object):
-        __slots__ = proplist
-
-    p = P()
-    for k, v in kwargs.iteritems():
-        setattr(p, k, v)
-    return p
-
 class MakeError(Exception):
     def __init__(self, message, loc=None):
         self.message = message
         self.loc = loc
 
     def __str__(self):
         locstr = ''
         if self.loc is not None:
@@ -82,8 +73,67 @@ else:
 try:
     from __builtin__ import any
 except ImportError:
     def any(it):
         for i in it:
             if i:
                 return True
         return False
+
+class _MostUsedItem(object):
+    __slots__ = ('key', 'o', 'count')
+
+    def __init__(self, key):
+        self.key = key
+        self.o = None
+        self.count = 1
+
+    def __repr__(self):
+        return "MostUsedItem(key=%r, count=%i, o=%r)" % (self.key, self.count, self.o)
+
+class MostUsedCache(object):
+    def __init__(self, capacity, creationfunc, verifyfunc):
+        self.capacity = capacity
+        self.cfunc = creationfunc
+        self.vfunc = verifyfunc
+
+        self.d = {}
+        self.active = [] # lazily sorted!
+
+    def setactive(self, item):
+        if item in self.active:
+            return
+
+        if len(self.active) == self.capacity:
+            self.active.sort(key=lambda i: i.count)
+            old = self.active.pop(0)
+            old.o = None
+            # print "Evicting %s" % old.key
+
+        self.active.append(item)
+
+    def get(self, key):
+        item = self.d.get(key, None)
+        if item is None:
+            item = _MostUsedItem(key)
+            self.d[key] = item
+        else:
+            item.count += 1
+
+        if item.o is not None and self.vfunc(key, item.o):
+            return item.o
+
+        item.o = self.cfunc(key)
+        self.setactive(item)
+        return item.o
+
+    def verify(self):
+        for k, v in self.d.iteritems():
+            if v.o:
+                assert v in self.active
+            else:
+                assert v not in self.active
+
+    def debugitems(self):
+        l = [i.key for i in self.active]
+        l.sort()
+        return l
--- a/build/pymake/tests/datatests.py
+++ b/build/pymake/tests/datatests.py
@@ -36,10 +36,43 @@ class GetPatSubstTest(unittest.TestCase)
     def runTest(self):
         for s, r, d, e in self.testdata:
             words = d.split()
             p = pymake.data.Pattern(s)
             a = ' '.join((p.subst(r, word, False)
                           for word in words))
             self.assertEqual(a, e, 'Pattern(%r).subst(%r, %r)' % (s, r, d))
 
+class LRUTest(unittest.TestCase):
+    # getkey, expected, funccount, debugitems
+    expected = (
+        (0, '', 1, (0,)),
+        (0, '', 2, (0,)),
+        (1, ' ', 3, (1, 0)),
+        (1, ' ', 3, (1, 0)),
+        (0, '', 4, (0, 1)),
+        (2, '  ', 5, (2, 0, 1)),
+        (1, ' ', 5, (1, 2, 0)),
+        (3, '   ', 6, (3, 1, 2)),
+    )
+
+    def spaceFunc(self, l):
+        self.funccount += 1
+        return ''.ljust(l)
+
+    def runTest(self):
+        self.funccount = 0
+        c = pymake.util.LRUCache(3, self.spaceFunc, lambda k, v: k % 2)
+        self.assertEqual(tuple(c.debugitems()), ())
+
+        for i in xrange(0, len(self.expected)):
+            k, e, fc, di = self.expected[i]
+
+            v = c.get(k)
+            self.assertEqual(v, e)
+            self.assertEqual(self.funccount, fc,
+                             "funccount, iteration %i, got %i expected %i" % (i, self.funccount, fc))
+            goti = tuple(c.debugitems())
+            self.assertEqual(goti, di,
+                             "debugitems, iteration %i, got %r expected %r" % (i, goti, di))
+
 if __name__ == '__main__':
     unittest.main()