pymake/functions.py
author Benjamin Smedberg <benjamin@smedbergs.us>
Thu, 26 Feb 2009 20:32:19 -0500
branchresolve-perf
changeset 191 7a8dc41115d67e05b6da433da2f7685ca9941b02
parent 185 cefacc0cd00266a54bdb2464cdaa5f9b9d0f881a
child 203 b5ecea7cc212f33a83386160146c8ff7ae0a58c3
permissions -rw-r--r--
I noticed Expansion.resolve still comes up really high on perf charts. This patch makes it much easier to resolve expansions which are just literals, which is very common for variable names. Unfortunately, this makes the code a fair bit more complex, and doesn't help nearly as much as I'd like. I'm beginning to wonder if getting gmake performance parity is impossible, or at least improbable given the current architecture: but I can't think of an alternate architecture that is better.

"""
Makefile functions.
"""

import parser, data, util
import subprocess, os, logging
from globrelative import glob
from cStringIO import StringIO

log = logging.getLogger('pymake.data')

class Function(object):
    """
    An object that represents a function call. This class is always subclassed
    with the following methods and attributes:

    minargs = minimum # of arguments
    maxargs = maximum # of arguments (0 means unlimited)

    def resolve(self, makefile, variables, setting)
        Calls the function
        @yields strings
    """
    def __init__(self, loc):
        self._arguments = []
        self.loc = loc
        assert self.minargs > 0

    def __getitem__(self, key):
        return self._arguments[key]

    def setup(self):
        argc = len(self._arguments)

        if argc < self.minargs:
            raise data.DataError("Not enough arguments to function %s, requires %s" % (self.name, self.minargs), self.loc)

        assert self.maxargs == 0 or argc <= self.maxargs, "Parser screwed up, gave us too many args"

    def append(self, arg):
        assert isinstance(arg, (data.Expansion, data.StringExpansion))
        self._arguments.append(arg)

    def __len__(self):
        return len(self._arguments)

class VariableRef(Function):
    def __init__(self, loc, vname):
        self.loc = loc
        assert isinstance(vname, (data.Expansion, data.StringExpansion))
        self.vname = vname
        
    def setup(self):
        assert False, "Shouldn't get here"

    def resolve(self, makefile, variables, setting):
        vname = self.vname.resolvestr(makefile, variables, setting)
        if vname in setting:
            raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)

        flavor, source, value = variables.get(vname)
        if value is None:
            log.debug("%s: variable '%s' was not set" % (self.loc, vname))
            return ()

        return value.resolve(makefile, variables, setting + [vname])

class SubstitutionRef(Function):
    """$(VARNAME:.c=.o) and $(VARNAME:%.c=%.o)"""
    def __init__(self, loc, varname, substfrom, substto):
        self.loc = loc
        self.vname = varname
        self.substfrom = substfrom
        self.substto = substto

    def setup(self):
        assert False, "Shouldn't get here"

    def resolve(self, makefile, variables, setting):
        vname = self.vname.resolvestr(makefile, variables, setting)
        if vname in setting:
            raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)

        substfrom = self.substfrom.resolvestr(makefile, variables, setting)
        substto = self.substto.resolvestr(makefile, variables, setting)

        flavor, source, value = variables.get(vname)
        if value is None:
            log.debug("%s: variable '%s' was not set" % (self.loc, vname))
            return ()

        f = data.Pattern(substfrom)
        if not f.ispattern():
            f = data.Pattern('%' + substfrom)
            substto = '%' + substto

        return util.joiniter((f.subst(substto, word, False)
                              for word in value.resolvesplit(makefile, variables, setting + [vname])))

class SubstFunction(Function):
    name = 'subst'
    minargs = 3
    maxargs = 3

    def resolve(self, makefile, variables, setting):
        s = self._arguments[0].resolvestr(makefile, variables, setting)
        r = self._arguments[1].resolvestr(makefile, variables, setting)
        d = self._arguments[2].resolvestr(makefile, variables, setting)
        yield d.replace(s, r)

class PatSubstFunction(Function):
    name = 'patsubst'
    minargs = 3
    maxargs = 3

    def resolve(self, makefile, variables, setting):
        s = self._arguments[0].resolvestr(makefile, variables, setting)
        r = self._arguments[1].resolvestr(makefile, variables, setting)

        p = data.Pattern(s)
        return util.joiniter((p.subst(r, word, False)
                              for word in self._arguments[2].resolvesplit(makefile, variables, setting)))

class StripFunction(Function):
    name = 'strip'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        return util.joiniter(self._arguments[0].resolvesplit(makefile, variables, setting))

class FindstringFunction(Function):
    name = 'findstring'
    minargs = 2
    maxargs = 2

    def resolve(self, makefile, variables, setting):
        s = self._arguments[0].resolvestr(makefile, variables, setting)
        r = self._arguments[1].resolvestr(makefile, variables, setting)
        if r.find(s) == -1:
            return
        yield s

class FilterFunction(Function):
    name = 'filter'
    minargs = 2
    maxargs = 2

    def resolve(self, makefile, variables, setting):
        plist = [data.Pattern(p)
                 for p in self._arguments[0].resolvesplit(makefile, variables, setting)]

        return util.joiniter((w for w in self._arguments[1].resolvesplit(makefile, variables, setting)
                              if util.any((p.match(w) for p in plist))))

class FilteroutFunction(Function):
    name = 'filter-out'
    minargs = 2
    maxargs = 2

    def resolve(self, makefile, variables, setting):
        plist = [data.Pattern(p)
                 for p in self._arguments[0].resolvesplit(makefile, variables, setting)]

        return util.joiniter((w for w in self._arguments[1].resolvesplit(makefile, variables, setting)
                              if not util.any((p.match(w) for p in plist))))

class SortFunction(Function):
    name = 'sort'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        d = list(self._arguments[0].resolvesplit(makefile, variables, setting))
        d.sort()
        return util.joiniter(d)

class WordFunction(Function):
    name = 'word'
    minargs = 2
    maxargs = 2

    def resolve(self, makefile, variables, setting):
        n = self._arguments[0].resolvestr(makefile, variables, setting)
        # TODO: provide better error if this doesn't convert
        n = int(n)
        words = list(self._arguments[1].resolvesplit(makefile, variables, setting))
        if n < 1 or n > len(words):
            return
        yield words[n - 1]

class WordlistFunction(Function):
    name = 'wordlist'
    minargs = 3
    maxargs = 3

    def resolve(self, makefile, variables, setting):
        nfrom = self._arguments[0].resolvestr(makefile, variables, setting)
        nto = self._arguments[1].resolvestr(makefile, variables, setting)
        # TODO: provide better errors if this doesn't convert
        nfrom = int(nfrom)
        nto = int(nto)

        words = list(self._arguments[2].resolvesplit(makefile, variables, setting))

        if nfrom < 1:
            nfrom = 1
        if nto < 1:
            nto = 1

        return util.joiniter(words[nfrom - 1:nto])

class WordsFunction(Function):
    name = 'words'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        yield str(len(list(self._arguments[0].resolvesplit(makefile, variables, setting))))

class FirstWordFunction(Function):
    name = 'firstword'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        for j in self._arguments[0].resolvesplit(makefile, variables, setting):
            yield j
            return

class LastWordFunction(Function):
    name = 'lastword'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        last = None
        for j in self._arguments[0].resolvesplit(makefile, variables, setting):
            last = j
        if last is not None:
            yield last

def pathsplit(path, default='./'):
    """
    Splits a path into dirpart, filepart on the last slash. If there is no slash, dirpart
    is ./
    """
    dir, slash, file = util.strrpartition(path, '/')
    if dir == '':
        return default, file

    return dir + slash, file

class DirFunction(Function):
    name = 'dir'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        return util.joiniter((pathsplit(path)[0]
                              for path in self._arguments[0].resolvesplit(makefile, variables, setting)))

class NotDirFunction(Function):
    name = 'notdir'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        return util.joiniter((pathsplit(path)[1]
                              for path in self._arguments[0].resolvesplit(makefile, variables, setting)))

class SuffixFunction(Function):
    name = 'suffix'
    minargs = 1
    maxargs = 1

    @staticmethod
    def suffixes(words):
        for w in words:
            dir, file = pathsplit(w)
            base, dot, suffix = util.strrpartition(file, '.')
            if base != '':
                yield dot + suffix

    def resolve(self, makefile, variables, setting):
        return util.joiniter(self.suffixes(self._arguments[0].resolvesplit(makefile, variables, setting)))

class BasenameFunction(Function):
    name = 'basename'
    minargs = 1
    maxargs = 1

    @staticmethod
    def basenames(words):
        for w in words:
            dir, file = pathsplit(w, '')
            base, dot, suffix = util.strrpartition(file, '.')
            if dot == '':
                base = suffix

            yield dir + base

    def resolve(self, makefile, variables, setting):
        return util.joiniter(self.basenames(self._arguments[0].resolvesplit(makefile, variables, setting)))

class AddSuffixFunction(Function):
    name = 'addprefix'
    minargs = 2
    maxargs = 2

    def resolve(self, makefile, variables, setting):
        suffix = self._arguments[0].resolvestr(makefile, variables, setting)

        return util.joiniter((w + suffix for w in self._arguments[1].resolvesplit(makefile, variables, setting)))

class AddPrefixFunction(Function):
    name = 'addsuffix'
    minargs = 2
    maxargs = 2

    def resolve(self, makefile, variables, setting):
        prefix = self._arguments[0].resolvestr(makefile, variables, setting)

        return util.joiniter((prefix + w for w in self._arguments[1].resolvesplit(makefile, variables, setting)))

class JoinFunction(Function):
    name = 'join'
    minargs = 2
    maxargs = 2

    @staticmethod
    def iterjoin(l1, l2):
        for i in xrange(0, max(len(l1), len(l2))):
            i1 = i < len(l1) and l1[i] or ''
            i2 = i < len(l2) and l2[i] or ''
            yield i1 + i2

    def resolve(self, makefile, variables, setting):
        list1 = list(self._arguments[0].resolvesplit(makefile, variables, setting))
        list2 = list(self._arguments[1].resolvesplit(makefile, variables, setting))

        return util.joiniter(self.iterjoin(list1, list2))

class WildcardFunction(Function):
    name = 'wildcard'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        patterns = self._arguments[0].resolvesplit(makefile, variables, setting)

        return util.joiniter((x.replace('\\','/')
                              for p in patterns
                              for x in glob(makefile.workdir, p)))

class RealpathFunction(Function):
    name = 'realpath'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        return util.joiniter((os.path.realpath(os.path.join(makefile.workdir, path)).replace('\\', '/')
                              for path in self._arguments[0].resolvesplit(makefile, variables, setting)))

class AbspathFunction(Function):
    name = 'abspath'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        assert os.path.isabs(makefile.workdir)
        return util.joiniter((os.path.join(makefile.workdir, path).replace('\\', '/')
                              for path in self._arguments[0].resolvesplit(makefile, variables, setting)))

class IfFunction(Function):
    name = 'if'
    minargs = 1
    maxargs = 3

    def setup(self):
        Function.setup(self)
        self._arguments[0].lstrip()
        self._arguments[0].rstrip()

    def resolve(self, makefile, variables, setting):
        condition = self._arguments[0].resolvestr(makefile, variables, setting)

        if len(condition):
            return self._arguments[1].resolve(makefile, variables, setting)

        if len(self._arguments) > 2:
            return self._arguments[2].resolve(makefile, variables, setting)

        return ()

class OrFunction(Function):
    name = 'or'
    minargs = 1
    maxargs = 0

    def resolve(self, makefile, variables, setting):
        for arg in self._arguments:
            r = arg.resolvestr(makefile, variables, setting)
            if r != '':
                return (r,)

        return ()

class AndFunction(Function):
    name = 'and'
    minargs = 1
    maxargs = 0

    def resolve(self, makefile, variables, setting):
        r = ''

        for arg in self._arguments:
            r = arg.resolvestr(makefile, variables, setting)
            if r == '':
                return ()

        return r,

class ForEachFunction(Function):
    name = 'foreach'
    minargs = 3
    maxargs = 3

    def resolve(self, makefile, variables, setting):
        vname = self._arguments[0].resolvestr(makefile, variables, setting)
        e = self._arguments[2]

        v = data.Variables(parent=variables)
        firstword = True

        for w in self._arguments[1].resolvesplit(makefile, variables, setting):
            if firstword:
                firstword = False
            else:
                yield ' '

            v.set(vname, data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, w)
            for j in e.resolve(makefile, v, setting):
                yield j

class CallFunction(Function):
    name = 'call'
    minargs = 1
    maxargs = 0

    def resolve(self, makefile, variables, setting):
        vname = self._arguments[0].resolvestr(makefile, variables, setting)
        if vname in setting:
            raise data.DataError("Recursively setting variable '%s'" % (vname,))

        v = data.Variables(parent=variables)
        v.set('0', data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, vname)
        for i in xrange(1, len(self._arguments)):
            param = self._arguments[i].resolvestr(makefile, variables, setting)
            v.set(str(i), data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, param)

        flavor, source, e = variables.get(vname)

        if e is None:
            return ()

        if flavor == data.Variables.FLAVOR_SIMPLE:
            log.warning("%s: calling variable '%s' which is simply-expanded" % (self.loc, vname))

        # but we'll do it anyway
        return e.resolve(makefile, v, setting + [vname])

class ValueFunction(Function):
    name = 'value'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        varname = self._arguments[0].resolvestr(makefile, variables, setting)

        flavor, source, value = variables.get(varname, expand=False)
        if value is None:
            return

        yield value

class EvalFunction(Function):
    name = 'eval'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        if makefile.parsingfinished:
            # GNU make allows variables to be set by recursive expansion during
            # command execution. This seems really dumb to me, so I don't!
            raise data.DataError("$(eval) not allowed via recursive expansion after parsing is finished", self.loc)

        text = StringIO(self._arguments[0].resolvestr(makefile, variables, setting))
        stmts = parser.parsestream(text, 'evaluation from %s' % self.loc)
        stmts.execute(makefile)
        return ()

class OriginFunction(Function):
    name = 'origin'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        vname = self._arguments[0].resolvestr(makefile, variables, setting)

        flavor, source, value = variables.get(vname)
        if source is None:
            return 'undefined',

        if source == data.Variables.SOURCE_OVERRIDE:
            return 'override',

        if source == data.Variables.SOURCE_MAKEFILE:
            return 'file',

        if source == data.Variables.SOURCE_ENVIRONMENT:
            return 'environment',

        if source == data.Variables.SOURCE_COMMANDLINE:
            return 'command line',

        if source == data.Variables.SOURCE_AUTOMATIC:
            return 'automatic',

        assert False, "Unexpected source value: %s" % source

class FlavorFunction(Function):
    name = 'flavor'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        varname = self._arguments[0].resolvestr(makefile, variables, setting)
        
        flavor, source, value = variables.get(varname)
        if flavor is None:
            return 'undefined',

        if flavor == data.Variables.FLAVOR_RECURSIVE:
            return 'recursive',

        if flavor == data.Variables.FLAVOR_SIMPLE:
            return 'simple',

        assert False, "Neither simple nor recursive?"

class ShellFunction(Function):
    name = 'shell'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        #TODO: call this once up-front somewhere and save the result?
        shell, msys = util.checkmsyscompat()
        cline = self._arguments[0].resolvestr(makefile, variables, setting)

        log.debug("%s: running shell command '%s'" % (self.loc, cline))
        if msys:
            cline = [shell, "-c", cline]
        p = subprocess.Popen(cline, shell=not msys, stdout=subprocess.PIPE, cwd=makefile.workdir)
        stdout, stderr = p.communicate()

        stdout = stdout.replace('\r\n', '\n')
        if len(stdout) > 1 and stdout[-1] == '\n':
            stdout = stdout[:-1]
        stdout = stdout.replace('\n', ' ')

        yield stdout

class ErrorFunction(Function):
    name = 'error'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        v = self._arguments[0].resolvestr(makefile, variables, setting)
        raise data.DataError(v, self.loc)

class WarningFunction(Function):
    name = 'warning'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        v = self._arguments[0].resolvestr(makefile, variables, setting)
        log.warning(v)
        return ()

class InfoFunction(Function):
    name = 'info'
    minargs = 1
    maxargs = 1

    def resolve(self, makefile, variables, setting):
        v = self._arguments[0].resolvestr(makefile, variables, setting)
        log.info(v)
        return ()

functionmap = {
    'subst': SubstFunction,
    'patsubst': PatSubstFunction,
    'strip': StripFunction,
    'findstring': FindstringFunction,
    'filter': FilterFunction,
    'filter-out': FilteroutFunction,
    'sort': SortFunction,
    'word': WordFunction,
    'wordlist': WordlistFunction,
    'words': WordsFunction,
    'firstword': FirstWordFunction,
    'lastword': LastWordFunction,
    'dir': DirFunction,
    'notdir': NotDirFunction,
    'suffix': SuffixFunction,
    'basename': BasenameFunction,
    'addsuffix': AddSuffixFunction,
    'addprefix': AddPrefixFunction,
    'join': JoinFunction,
    'wildcard': WildcardFunction,
    'realpath': RealpathFunction,
    'abspath': AbspathFunction,
    'if': IfFunction,
    'or': OrFunction,
    'and': AndFunction,
    'foreach': ForEachFunction,
    'call': CallFunction,
    'value': ValueFunction,
    'eval': EvalFunction,
    'origin': OriginFunction,
    'flavor': FlavorFunction,
    'shell': ShellFunction,
    'error': ErrorFunction,
    'warning': WarningFunction,
    'info': InfoFunction,
}