Some more win32-compat changes win32-msys
authorTed Mielczarek <ted.mielczarek@gmail.com>
Fri, 20 Feb 2009 13:08:17 -0500
branchwin32-msys
changeset 144 bf4d1889ba7117450ce0daa9c21d4d0f2292a386
parent 119 680f9674f48b31d83f4f2996c1a6fb4b0d4e00d8
child 145 660f249d254dcebbd1ee456b464e0b66c255a7c6
push id81
push usertmielczarek@mozilla.com
push dateFri, 20 Feb 2009 18:11:31 +0000
Some more win32-compat changes * Add a 'returncode-on' setting to the test harness so we can fail tests per-platform. * Split the symlink tests out of file-functions.mk into file-functions-symlinks.mk, and fail that on Windows. * Use forward slashes consistently in filenames to be compatible with GNU Make on Windows. * Fix PatternRule to execute through the MSYS shell when running under MSYS. * Change automatic-variables.mk to not use 'mkdir -p src/subdir subd' as that breaks due to a MSYS mkdir bug. * Change include-notfound.mk, since MAKEFILE_LIST uses native win32 paths on GNU Make on Windows (even in MSYS) * Add a NATIVE_TESTPATH variable to the test harness. * Pass a __WIN32__ variable from the test harness when running on Win32.
make.py
pymake/data.py
pymake/functions.py
pymake/parser.py
tests/automatic-variables.mk
tests/file-functions-symlinks.mk
tests/file-functions.mk
tests/include-notfound.mk
tests/runtests.py
--- a/make.py
+++ b/make.py
@@ -124,17 +124,18 @@ if len(options.makefiles) == 0:
         print "No makefile found"
         sys.exit(2)
 
 try:
     def parse():
         i = 0
 
         while True:
-            m = Makefile(restarts=i, make='%s %s' % (sys.executable, sys.argv[0]),
+            m = Makefile(restarts=i, make='%s %s' % (sys.executable.replace('\\','/'),
+                                                     sys.argv[0].replace('\\', '/')),
                          makeflags=makeflags, makelevel=makelevel)
 
             starttime = time.time()
             targets = parsecommandlineargs(m, arguments)
             for f in options.makefiles:
                 m.include(f)
 
             log.info("Parsing[%i] took %f seconds" % (i, time.time() - starttime,))
--- a/pymake/data.py
+++ b/pymake/data.py
@@ -78,17 +78,16 @@ def checkmsyscompat():
         shell = os.environ['SHELL']
     elif 'COMSPEC' in os.environ:
         shell = os.environ['COMSPEC']
     else:
         raise DataError("Can't find a suitable shell!")
 
     prependshell = False
     if 'MSYSTEM' in os.environ and os.environ['MSYSTEM'] == 'MINGW32':
-        #XXX: transliterate slashes?
         prependshell = True
         if not shell.lower().endswith(".exe"):
             shell += ".exe"
     return (shell, prependshell)
 
 class Expansion(object):
     """
     A representation of expanded data, such as that for a recursively-expanded variable, a command, etc.
@@ -644,18 +643,17 @@ class Target(object):
                                 return
 
                     self.vpathtarget = self.target
                     self.mtime = None
                     return
 
         search = [self.target]
         if not os.path.isabs(self.target):
-            #XXX: not using os.path.join for MSYS support :-/
-            search += [dir + "/" + self.target
+            search += [os.path.join(dir, self.target).replace('\\','/')
                        for dir in makefile.vpath]
 
         for t in search:
             mtime = getmtime(t)
             if mtime is not None:
                 self.vpathtarget = t
                 self.mtime = mtime
                 return
@@ -944,25 +942,28 @@ class PatternRule(object):
 
         v = Variables(parent=target.variables)
         setautomaticvariables(v, makefile, target,
                               self.prerequisitesforstem(dir, stem))
         setautomatic(v, '*', [dir + stem])
 
         env = makefile.getsubenvironment(v)
 
+        shell, prependshell = checkmsyscompat()
         for c in self.commands:
             cstring = c.resolve(v)
             for cline in splitcommand(cstring):
                 cline, isHidden, isRecursive, ignoreErrors = findmodifiers(cline)
                 if not len(cline) or cline.isspace():
                     continue
                 if not isHidden:
                     print "%s $ %s" % (c.loc, cline)
-                r = subprocess.call(cline, shell=True, env=env)
+                if prependshell:
+                    cline = [shell, "-c", cline]
+                r = subprocess.call(cline, shell=not prependshell, env=env)
                 if r != 0 and not ignoreErrors:
                     raise DataError("command '%s' failed, return code was %s" % (cline, r), c.loc)
 
 class Makefile(object):
     def __init__(self, workdir=None, restarts=0, make=None, makeflags=None, makelevel=0):
         self.defaulttarget = None
 
         self.variables = Variables()
@@ -975,17 +976,17 @@ class Makefile(object):
         self.implicitrules = []
         self.parsingfinished = False
 
         if workdir is None:
             workdir = os.getcwd()
         workdir = os.path.realpath(workdir)
         self.workdir = workdir
         self.variables.set('CURDIR', Variables.FLAVOR_SIMPLE,
-                           Variables.SOURCE_AUTOMATIC, workdir)
+                           Variables.SOURCE_AUTOMATIC, workdir.replace('\\','/'))
 
         # the list of included makefiles, whether or not they existed
         self.included = []
 
         self.variables.set('MAKE_RESTARTS', Variables.FLAVOR_SIMPLE,
                            Variables.SOURCE_AUTOMATIC, restarts > 0 and str(restarts) or '')
 
         if make is not None:
--- a/pymake/functions.py
+++ b/pymake/functions.py
@@ -356,36 +356,36 @@ class JoinFunction(Function):
 class WildcardFunction(Function):
     name = 'wildcard'
     minargs = 1
     maxargs = 1
 
     def resolve(self, variables, setting):
         # TODO: will need work when we support -C without actually changing the OS cwd
         pattern = self._arguments[0].resolve(variables, setting)
-        return ' '.join(glob.glob(pattern))
+        return ' '.join([x.replace('\\','/') for x in glob.glob(pattern)])
 
 class RealpathFunction(Function):
     name = 'realpath'
     minargs = 1
     maxargs = 1
 
     def resolve(self, variables, setting):
         # TODO: will need work when we support -C without actually changing the OS cwd
-        return ' '.join((os.path.realpath(f)
+        return ' '.join((os.path.realpath(f).replace('\\','/')
                          for f in data.splitwords(self._arguments[0].resolve(variables, setting))))
 
 class AbspathFunction(Function):
     name = 'abspath'
     minargs = 1
     maxargs = 1
 
     def resolve(self, variables, setting):
         # TODO: will need work when we support -C without actually changing the OS cwd
-        return ' '.join((os.path.abspath(f)
+        return ' '.join((os.path.abspath(f).replace('\\','/')
                          for f in data.splitwords(self._arguments[0].resolve(variables, setting))))
 
 class IfFunction(Function):
     name = 'if'
     minargs = 1
     maxargs = 3
 
     def setup(self):
--- a/pymake/parser.py
+++ b/pymake/parser.py
@@ -652,17 +652,18 @@ def parsestream(fd, filename, makefile):
                             skipwhitespace=False)
 
                 continue
 
             if kword in ('include', '-include'):
                 incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
                 files = data.splitwords(incfile.resolve(makefile.variables))
                 for f in files:
-                    makefile.include(f, kword == 'include', loc=d.getloc(offset))
+                    makefile.include(f.replace('\\','/'),
+                                     kword == 'include', loc=d.getloc(offset))
                 continue
 
             if kword == 'override':
                 e, token, offset = parsemakesyntax(d, offset, varsettokens, itermakefilechars)
                 e.lstrip()
                 e.rstrip()
 
                 if token is None:
--- a/tests/automatic-variables.mk
+++ b/tests/automatic-variables.mk
@@ -1,10 +1,11 @@
 $(shell \
-mkdir -p src/subd subd; \
+mkdir -p src/subd; \
+mkdir subd; \
 touch dummy; \
 sleep 1; \
 touch subd/test.out src/subd/test.in2; \
 sleep 1; \
 touch subd/test.out2 src/subd/test.in; \
 sleep 1; \
 touch subd/host_test.out subd/host_test.out2; \
 sleep 1; \
new file mode 100644
--- /dev/null
+++ b/tests/file-functions-symlinks.mk
@@ -0,0 +1,21 @@
+#T returncode-on: {'win32': 2}
+$(shell \
+touch test.file; \
+ln -s test.file test.symlink; \
+touch .testhidden; \
+mkdir foo; \
+touch foo/testfile; \
+)
+
+all:
+	test "$(abspath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.symlink"
+	test "$(realpath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.file"
+	test "$(sort $(wildcard *))" = "foo test.file test.symlink"
+# commented out because GNU make matches . and .. while python doesn't, and I don't
+# care enough
+#	test "$(sort $(wildcard .*))" = ". .. .testhidden"
+	test "$(sort $(wildcard test*))" = "test.file test.symlink"
+	test "$(sort $(wildcard foo/*))" = "foo/testfile"
+	test "$(sort $(wildcard ./*))" = "./foo ./test.file ./test.symlink"
+	test "$(sort $(wildcard f?o/*))" = "foo/testfile"
+	@echo TEST-PASS
--- a/tests/file-functions.mk
+++ b/tests/file-functions.mk
@@ -1,20 +1,19 @@
 $(shell \
 touch test.file; \
-ln -s test.file test.symlink; \
 touch .testhidden; \
 mkdir foo; \
 touch foo/testfile; \
 )
 
 all:
-	test "$(abspath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.symlink"
-	test "$(realpath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.file"
-	test "$(sort $(wildcard *))" = "foo test.file test.symlink"
+	test "$(abspath test.file)" = "$(CURDIR)/test.file"
+	test "$(realpath test.file)" = "$(CURDIR)/test.file"
+	test "$(sort $(wildcard *))" = "foo test.file"
 # commented out because GNU make matches . and .. while python doesn't, and I don't
 # care enough
 #	test "$(sort $(wildcard .*))" = ". .. .testhidden"
-	test "$(sort $(wildcard test*))" = "test.file test.symlink"
+	test "$(sort $(wildcard test*))" = "test.file"
 	test "$(sort $(wildcard foo/*))" = "foo/testfile"
-	test "$(sort $(wildcard ./*))" = "./foo ./test.file ./test.symlink"
+	test "$(sort $(wildcard ./*))" = "./foo ./test.file"
 	test "$(sort $(wildcard f?o/*))" = "foo/testfile"
 	@echo TEST-PASS
--- a/tests/include-notfound.mk
+++ b/tests/include-notfound.mk
@@ -1,13 +1,19 @@
-ifneq ($(strip $(MAKEFILE_LIST)),$(TESTPATH)/include-notfound.mk)
-$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)')
+ifdef __WIN32__
+PS:=\\#
+else
+PS:=/
+endif
+
+ifneq ($(strip $(MAKEFILE_LIST)),$(NATIVE_TESTPATH)$(PS)include-notfound.mk)
+$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)' (expected '$(NATIVE_TESTPATH)$(PS)include-notfound.mk'))
 endif
 
 -include notfound.inc-dummy
 
-ifneq ($(strip $(MAKEFILE_LIST)),$(TESTPATH)/include-notfound.mk)
-$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)')
+ifneq ($(strip $(MAKEFILE_LIST)),$(NATIVE_TESTPATH)$(PS)include-notfound.mk)
+$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)' (expected '$(NATIVE_TESTPATH)$(PS)include-notfound.mk'))
 endif
 
 all:
 	@echo TEST-PASS
 
--- a/tests/runtests.py
+++ b/tests/runtests.py
@@ -5,16 +5,17 @@ walk the directory for files named .mk a
 For each test, we run gmake -f test.mk. By default, make must exit with an exit code of 0, and must print 'TEST-PASS'.
 
 Each test is run in an empty directory.
 
 The test file may contain lines at the beginning to alter the default behavior. These are all evaluated as python:
 
 #T commandline: ['extra', 'params', 'here']
 #T returncode: 2
+#T returncode-on: {'win32': 2}
 """
 
 from subprocess import Popen, PIPE, STDOUT
 from optparse import OptionParser
 import os, re, sys, shutil
 
 thisdir = os.path.dirname(os.path.abspath(__file__))
 
@@ -36,38 +37,51 @@ for a in args:
         for path, dirnames, filenames in os.walk(a):
             for f in filenames:
                 if f.endswith('.mk'):
                     makefiles.append('%s/%s' % (path, f))
     else:
         print >>sys.stderr, "Error: Unknown file on command line"
         sys.exit(1)
 
-tre = re.compile('^#T ([a-z]+): (.*)$')
+tre = re.compile('^#T ([a-z-]+): (.*)$')
 
 for makefile in makefiles:
     print "Testing: %s" % makefile,
 
     if os.path.exists(opts.tempdir): shutil.rmtree(opts.tempdir)
     os.mkdir(opts.tempdir, 0755)
 
-    cline = [opts.make, '-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir]
+    # For some reason, MAKEFILE_LIST uses native paths in GNU make on Windows
+    # (even in MSYS!) so we pass both TESTPATH and NATIVE_TESTPATH
+    cline = [opts.make, '-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir.replace('\\','/'), 'NATIVE_TESTPATH=%s' % thisdir]
+    if sys.platform == 'win32':
+        #XXX: hack to run pymake on windows
+        if opts.make.endswith('.py'):
+            cline = [sys.executable] + cline
+        #XXX: hack so we can specialize the separator character on windows.
+        # we really shouldn't need this, but y'know
+        cline += ['__WIN32__=1']
+        
     returncode = 0
 
     mdata = open(makefile)
     for line in mdata:
         m = tre.search(line)
         if m is None:
             break
         key, data = m.group(1, 2)
         data = eval(data)
         if key == 'commandline':
             cline.extend(data)
         elif key == 'returncode':
             returncode = data
+        elif key == 'returncode-on':
+            if sys.platform in data:
+                returncode = data[sys.platform]
         else:
             print >>sys.stderr, "Unexpected #T key: %s" % key
             sys.exit(1)
 
     mdata.close()
 
     p = Popen(cline, stdout=PIPE, stderr=STDOUT)
     stdout, d = p.communicate()