bug 1481612 - Add a --with-windows-wheel option to mach vendor python. r=gps
authorTed Mielczarek <ted@mielczarek.org>
Wed, 10 Oct 2018 19:54:36 +0000
changeset 496343 60407ad1392236779a537033965c59e3170416f3
parent 496342 aafdbf2213ec4463e830f810dbb71016f5e49c5f
child 496344 7243432584616e49d01a0b8910518b359d5aab99
push id9984
push userffxbld-merge
push dateMon, 15 Oct 2018 21:07:35 +0000
treeherdermozilla-beta@183d27ea8570 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1481612
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1481612 - Add a --with-windows-wheel option to mach vendor python. r=gps This option is very single-purpose: it's intended to let us vendor an unpacked wheel for psutil on Windows. To that end the mach command will error if you try to use it for anything but vendoring a single package. The mach command will vendor source packages as it currently does, and then run `pip download` again with some hardcoded parameters to fetch the right wheel for Python 2.7 on win64 and unpack it to a `package-platform` directory under `third_party/python`. I don't expect this to be used for anything but psutil, but it should make life simpler for anyone that wants to update our vendored copy of psutil in the future. Differential Revision: https://phabricator.services.mozilla.com/D3435
python/mozbuild/mozbuild/mach_commands.py
python/mozbuild/mozbuild/vendor_python.py
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -2697,16 +2697,19 @@ class Vendor(MachCommandBase):
         default=False)
     def vendor_aom(self, **kwargs):
         from mozbuild.vendor_aom import VendorAOM
         vendor_command = self._spawn(VendorAOM)
         vendor_command.vendor(**kwargs)
 
     @SubCommand('vendor', 'python',
                 description='Vendor Python packages from pypi.org into third_party/python')
+    @CommandArgument('--with-windows-wheel', action='store_true',
+        help='Vendor a wheel for Windows along with the source package',
+        default=False)
     @CommandArgument('packages', default=None, nargs='*', help='Packages to vendor. If omitted, packages and their dependencies defined in Pipfile.lock will be vendored. If Pipfile has been modified, then Pipfile.lock will be regenerated. Note that transient dependencies may be updated when running this command.')
     def vendor_python(self, **kwargs):
         from mozbuild.vendor_python import VendorPython
         vendor_command = self._spawn(VendorPython)
         vendor_command.vendor(**kwargs)
 
     @SubCommand('vendor', 'manifest',
                 description='Vendor externally hosted repositories into this '
--- a/python/mozbuild/mozbuild/vendor_python.py
+++ b/python/mozbuild/mozbuild/vendor_python.py
@@ -12,24 +12,26 @@ import mozfile
 import mozpack.path as mozpath
 from mozbuild.base import MozbuildObject
 from mozfile import NamedTemporaryFile, TemporaryDirectory
 from mozpack.files import FileFinder
 
 
 class VendorPython(MozbuildObject):
 
-    def vendor(self, packages=None):
+    def vendor(self, packages=None, with_windows_wheel=False):
         self.populate_logger()
         self.log_manager.enable_unstructured()
 
         vendor_dir = mozpath.join(
             self.topsrcdir, os.path.join('third_party', 'python'))
 
         packages = packages or []
+        if with_windows_wheel and len(packages) != 1:
+            raise Exception('--with-windows-wheel is only supported for a single package!')
 
         self._activate_virtualenv()
         pip_compile = os.path.join(self.virtualenv_manager.bin_path, 'pip-compile')
         if not os.path.exists(pip_compile):
             path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party', 'python', 'pip-tools'))
             self.virtualenv_manager.install_pip_package(path, vendored=True)
         spec = os.path.join(vendor_dir, 'requirements.in')
         requirements = os.path.join(vendor_dir, 'requirements.txt')
@@ -51,16 +53,31 @@ class VendorPython(MozbuildObject):
                 # use requirements.txt to download archived source distributions of all packages
                 self.virtualenv_manager._run_pip([
                     'download',
                     '-r', requirements,
                     '--no-deps',
                     '--dest', tmp,
                     '--no-binary', ':all:',
                     '--disable-pip-version-check'])
+                if with_windows_wheel:
+                    # This is hardcoded to CPython 2.7 for win64, which is good
+                    # enough for what we need currently. If we need psutil for Python 3
+                    # in the future that coudl be added here as well.
+                    self.virtualenv_manager._run_pip([
+                        'download',
+                        '--dest', tmp,
+                        '--no-deps',
+                        '--only-binary', ':all:',
+                        '--platform', 'win_amd64',
+                        '--implementation', 'cp',
+                        '--python-version', '27',
+                        '--abi', 'none',
+                        '--disable-pip-version-check',
+                        packages[0]])
                 self._extract(tmp, vendor_dir)
 
             shutil.copyfile(tmpspec.name, spec)
             self.repository.add_remove_files(vendor_dir)
 
     def _update_packages(self, spec, packages):
         for package in packages:
             if not all(package.partition('==')):
@@ -78,13 +95,28 @@ class VendorPython(MozbuildObject):
         with open(spec, 'w') as f:
             for name, version in sorted(requirements.items()):
                 f.write('{}=={}\n'.format(name, version))
 
     def _extract(self, src, dest):
         """extract source distribution into vendor directory"""
         finder = FileFinder(src)
         for path, _ in finder.find('*'):
-            # packages extract into package-version directory name and we strip the version
-            tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
-            target = os.path.join(dest, tld.rpartition('-')[0])
-            mozfile.remove(target)  # remove existing version of vendored package
-            mozfile.move(tld, target)
+            base, ext = os.path.splitext(path)
+            if ext == '.whl':
+                # Wheels would extract into a directory with the name of the package, but
+                # we want the platform signifiers, minus the version number.
+                # Wheel filenames look like:
+                # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}
+                bits = base.split('-')
+
+                # Remove the version number.
+                bits.pop(1)
+                target = os.path.join(dest, '-'.join(bits))
+                mozfile.remove(target)  # remove existing version of vendored package
+                os.mkdir(target)
+                mozfile.extract(os.path.join(finder.base, path), target)
+            else:
+                # packages extract into package-version directory name and we strip the version
+                tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
+                target = os.path.join(dest, tld.rpartition('-')[0])
+                mozfile.remove(target)  # remove existing version of vendored package
+                mozfile.move(tld, target)