servo: Merge #14039 - Fix test-wpt and test-css for Windows (from metajack:windows-wpt); r=jgraham
authorJack Moffitt <jack@metajack.im>
Wed, 16 Nov 2016 13:41:27 -0600
changeset 389138 914e8b90e3cbfe8bc91ffe96d0a85e1a93eb4a83
parent 389137 af131bf9d7582fa8988664cf02cb39ed19763d30
child 389139 8ef847a24d3d382f614259e3333b1d849207faa8
push id1468
push userasasaki@mozilla.com
push dateMon, 05 Jun 2017 19:31:07 +0000
treeherdermozilla-release@0641fc6ee9d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
servo: Merge #14039 - Fix test-wpt and test-css for Windows (from metajack:windows-wpt); r=jgraham <!-- Please describe your changes on the following line: --> In addition to minor changes for Windows, this forces Windows Python to be used for all Windows builds (instead of using Windows Python only for pc-windows-msvc builds). --- <!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: --> - [x] `./mach build -d` does not report any errors - [x] `./mach test-tidy` does not report any errors - [ ] These changes fix #__ (github issue number if applicable). <!-- Either: --> - [ ] There are tests for these changes OR - [x] These changes do not require tests because testing the tests is too meta for me <!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. --> Source-Repo: https://github.com/servo/servo Source-Revision: 17bf6aed21cb966c52382e7cef23d2bd66144874
servo/README.md
servo/appveyor.yml
servo/mach
servo/python/mach_bootstrap.py
servo/python/servo/command_base.py
--- a/servo/README.md
+++ b/servo/README.md
@@ -130,26 +130,24 @@ pacman -Sy git mingw-w64-x86_64-toolchai
 export GCC_URL=http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc
 export GCC_EXT=5.4.0-1-any.pkg.tar.xz
 pacman -U --noconfirm $GCC_URL-$GCC_EXT $GCC_URL-ada-$GCC_EXT \
     $GCC_URL-fortran-$GCC_EXT $GCC_URL-libgfortran-$GCC_EXT $GCC_URL-libs-$GCC_EXT \
     $GCC_URL-objc-$GCC_EXT
 easy_install-2.7 pip virtualenv
 ```
 
-Open a new MSYS shell window as Administrator and remove the Python binaries (they
-are not compatible with our `mach` driver script yet, unfortunately):
+Add the following line to the end of `.profile` in your home directory:
 
-```sh
-cd /mingw64/bin
-mv python2.exe python2-mingw64.exe
-mv python2.7.exe python2.7-mingw64.exe
+```
+export PATH=/c/Python27:$PATH
 ```
 
-Now, open a MINGW64 (not MSYS!) shell window, and you should be able to build servo as usual!
+Now, open a MINGW64 (not MSYS!) shell window, and you should be able to build
+servo as usual!
 
 #### Cross-compilation for Android
 
 Pre-installed Android tools are needed. See wiki for
 [details](https://github.com/servo/servo/wiki/Building-for-Android)
 
 ## The Rust compiler
 
--- a/servo/appveyor.yml
+++ b/servo/appveyor.yml
@@ -63,11 +63,11 @@ install:
 #init:
 #  - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
 #
 #on_finish:
 #  - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
 
 build_script:
   - if %BUILD_ENV%==msvc mach build -d -v && mach test-unit
-  - if %BUILD_ENV%==gnu  bash -lc "cd $APPVEYOR_BUILD_FOLDER; ./mach build -d -v && ./mach test-unit"
+  - if %BUILD_ENV%==gnu  bash -lc "export PATH=/c/Python27:$PATH; cd $APPVEYOR_BUILD_FOLDER; ./mach build -d -v && ./mach test-unit"
 
 test: off
--- a/servo/mach
+++ b/servo/mach
@@ -1,25 +1,92 @@
 #!/bin/sh
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # The beginning of this script is both valid shell and valid python,
 # such that the script starts with the shell and is reexecuted with
 # the right python.
-'''which' python2.7 > /dev/null 2> /dev/null && exec python2.7 "$0" "$@" || exec python "$0" "$@"
+''':' && if [ ! -z "$MSYSTEM" ] ; then exec python "$0" "$@" ; else which python2.7 > /dev/null 2> /dev/null && exec python2.7 "$0" "$@" || exec python "$0" "$@" ; fi
 '''
 
 from __future__ import print_function, unicode_literals
 
+
 import os
 import sys
 
+
 def main(args):
     topdir = os.path.abspath(os.path.dirname(sys.argv[0]))
     sys.path.insert(0, os.path.join(topdir, "python"))
     import mach_bootstrap
     mach = mach_bootstrap.bootstrap(topdir)
     sys.exit(mach.run(sys.argv[1:]))
 
+
 if __name__ == '__main__':
+    if sys.platform == 'win32':
+        # This is a complete hack to work around the fact that Windows
+        # multiprocessing needs to import the original module (ie: this
+        # file), but only works if it has a .py extension.
+        #
+        # We do this by a sort of two-level function interposing. The first
+        # level interposes forking.get_command_line() with our version defined
+        # in my_get_command_line(). Our version of get_command_line will
+        # replace the command string with the contents of the fork_interpose()
+        # function to be used in the subprocess.
+        #
+        # The subprocess then gets an interposed imp.find_module(), which we
+        # hack up to find 'mach' without the .py extension, since we already
+        # know where it is (it's us!). If we're not looking for 'mach', then
+        # the original find_module will suffice.
+        #
+        # See also: http://bugs.python.org/issue19946
+        # And: https://bugzilla.mozilla.org/show_bug.cgi?id=914563
+        import inspect
+        from multiprocessing import forking
+        global orig_command_line
+
+        def fork_interpose():
+            import imp
+            import os
+            import sys
+            orig_find_module = imp.find_module
+            def my_find_module(name, dirs):
+                if name == 'mach':
+                    path = os.path.join(dirs[0], 'mach')
+                    f = open(path)
+                    return (f, path, ('', 'r', imp.PY_SOURCE))
+                return orig_find_module(name, dirs)
+
+            # Don't allow writing bytecode file for mach module.
+            orig_load_module = imp.load_module
+            def my_load_module(name, file, path, description):
+                # multiprocess.forking invokes imp.load_module manually and
+                # hard-codes the name __parents_main__ as the module name.
+                if name == '__parents_main__':
+                    old_bytecode = sys.dont_write_bytecode
+                    sys.dont_write_bytecode = True
+                    try:
+                        return orig_load_module(name, file, path, description)
+                    finally:
+                        sys.dont_write_bytecode = old_bytecode
+
+                return orig_load_module(name, file, path, description)
+
+            imp.find_module = my_find_module
+            imp.load_module = my_load_module
+            from multiprocessing.forking import main; main()
+
+        def my_get_command_line():
+            fork_code, lineno = inspect.getsourcelines(fork_interpose)
+            # Remove the first line (for 'def fork_interpose():') and the three
+            # levels of indentation (12 spaces).
+            fork_string = ''.join(x[12:] for x in fork_code[1:])
+            cmdline = orig_command_line()
+            cmdline[2] = fork_string
+            return cmdline
+        orig_command_line = forking.get_command_line
+        forking.get_command_line = my_get_command_line
+
     main(sys.argv)
--- a/servo/python/mach_bootstrap.py
+++ b/servo/python/mach_bootstrap.py
@@ -71,19 +71,26 @@ CATEGORIES = {
         'short': 'Disabled',
         'long': 'The disabled commands are hidden by default. Use -v to display them. These commands are unavailable '
                 'for your current context, run "mach <command>" to see why.',
         'priority': 0,
     }
 }
 
 # Possible names of executables
-PYTHON_NAMES = ["python-2.7", "python2.7", "python2", "python"]
-VIRTUALENV_NAMES = ["virtualenv-2.7", "virtualenv2.7", "virtualenv2", "virtualenv"]
-PIP_NAMES = ["pip-2.7", "pip2.7", "pip2", "pip"]
+# NOTE: Windows Python doesn't provide versioned executables, so we must use
+# the plain names. On MSYS, we still use Windows Python.
+if sys.platform in ['msys', 'win32']:
+    PYTHON_NAMES = ["python"]
+    VIRTUALENV_NAMES = ["virtualenv"]
+    PIP_NAMES = ["pip"]
+else:
+    PYTHON_NAMES = ["python-2.7", "python2.7", "python2", "python"]
+    VIRTUALENV_NAMES = ["virtualenv-2.7", "virtualenv2.7", "virtualenv2", "virtualenv"]
+    PIP_NAMES = ["pip-2.7", "pip2.7", "pip2", "pip"]
 
 
 def _get_exec_path(names, is_valid_path=lambda _path: True):
     for name in names:
         path = find_executable(name)
         if path and is_valid_path(path):
             return path
     return None
@@ -192,20 +199,21 @@ def bootstrap(topdir):
     # https://github.com/servo/servo/issues/9442
     if ' ' in topdir:
         print('Cannot run mach in a path with spaces.')
         print('Current path:', topdir)
         sys.exit(1)
 
     # We don't support MinGW Python
     if os.path.join(os.sep, 'mingw64', 'bin') in sys.executable:
-        print('Cannot run mach with MinGW Python.')
-        print('\nPlease rename following files:')
-        print(' /mingw64/bin/python2.exe   -> /mingw64/bin/python2-mingw64.exe')
-        print(' /mingw64/bin/python2.7.exe -> /mingw64/bin/python2.7-mingw64.exe')
+        print('Cannot run mach with MinGW or MSYS Python.')
+        print('\nPlease add the path to Windows Python (usually /c/Python27) to your path.')
+        print('You can do this by appending the line:')
+        print('    export PATH=/c/Python27:$PATH')
+        print('to your ~/.profile.')
         sys.exit(1)
 
     # Ensure we are running Python 2.7+. We put this check here so we generate a
     # user-friendly error message rather than a cryptic stack trace on module import.
     if not (3, 0) > sys.version_info >= (2, 7):
         print('Python 2.7 or above (but not Python 3) is required to run mach.')
         print('You are running Python', platform.python_version())
         sys.exit(1)
--- a/servo/python/servo/command_base.py
+++ b/servo/python/servo/command_base.py
@@ -102,17 +102,20 @@ def host_triple():
     os_type = platform.system().lower()
     if os_type == "linux":
         os_type = "unknown-linux-gnu"
     elif os_type == "darwin":
         os_type = "apple-darwin"
     elif os_type == "android":
         os_type = "linux-androideabi"
     elif os_type == "windows":
-        os_type = "pc-windows-msvc"
+        if os.getenv("MSYSTEM") is None:
+            os_type = "pc-windows-msvc"
+        else:
+            os_type = "pc-windows-gnu"
     elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
         os_type = "pc-windows-gnu"
     elif os_type == "freebsd":
         os_type = "unknown-freebsd"
     else:
         os_type = "unknown"
 
     cpu_type = platform.machine().lower()