python/mozbuild/mozbuild/vendor_python.py
author Ted Mielczarek <ted@mielczarek.org>
Wed, 10 Oct 2018 19:54:36 +0000
changeset 499055 60407ad1392236779a537033965c59e3170416f3
parent 498965 1a9cbc785296806682eb165e0307b05b8f45b7e7
child 499058 f5c1a7734493d3cf9e1ded5731bba0528b01155f
permissions -rw-r--r--
bug 1481612 - Add a --with-windows-wheel option to mach vendor python. r=gps This option is very single-purpose: it's intended to let us vendor an unpacked wheel for psutil on Windows. To that end the mach command will error if you try to use it for anything but vendoring a single package. The mach command will vendor source packages as it currently does, and then run `pip download` again with some hardcoded parameters to fetch the right wheel for Python 2.7 on win64 and unpack it to a `package-platform` directory under `third_party/python`. I don't expect this to be used for anything but psutil, but it should make life simpler for anyone that wants to update our vendored copy of psutil in the future. Differential Revision: https://phabricator.services.mozilla.com/D3435

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

import os
import shutil
import subprocess

import mozfile
import mozpack.path as mozpath
from mozbuild.base import MozbuildObject
from mozfile import NamedTemporaryFile, TemporaryDirectory
from mozpack.files import FileFinder


class VendorPython(MozbuildObject):

    def vendor(self, packages=None, with_windows_wheel=False):
        self.populate_logger()
        self.log_manager.enable_unstructured()

        vendor_dir = mozpath.join(
            self.topsrcdir, os.path.join('third_party', 'python'))

        packages = packages or []
        if with_windows_wheel and len(packages) != 1:
            raise Exception('--with-windows-wheel is only supported for a single package!')

        self._activate_virtualenv()
        pip_compile = os.path.join(self.virtualenv_manager.bin_path, 'pip-compile')
        if not os.path.exists(pip_compile):
            path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party', 'python', 'pip-tools'))
            self.virtualenv_manager.install_pip_package(path, vendored=True)
        spec = os.path.join(vendor_dir, 'requirements.in')
        requirements = os.path.join(vendor_dir, 'requirements.txt')

        with NamedTemporaryFile('w') as tmpspec:
            shutil.copyfile(spec, tmpspec.name)
            self._update_packages(tmpspec.name, packages)

            # resolve the dependencies and update requirements.txt
            subprocess.check_output([
                pip_compile,
                tmpspec.name,
                '--no-header',
                '--no-index',
                '--output-file', requirements,
                '--generate-hashes'])

            with TemporaryDirectory() as tmp:
                # use requirements.txt to download archived source distributions of all packages
                self.virtualenv_manager._run_pip([
                    'download',
                    '-r', requirements,
                    '--no-deps',
                    '--dest', tmp,
                    '--no-binary', ':all:',
                    '--disable-pip-version-check'])
                if with_windows_wheel:
                    # This is hardcoded to CPython 2.7 for win64, which is good
                    # enough for what we need currently. If we need psutil for Python 3
                    # in the future that coudl be added here as well.
                    self.virtualenv_manager._run_pip([
                        'download',
                        '--dest', tmp,
                        '--no-deps',
                        '--only-binary', ':all:',
                        '--platform', 'win_amd64',
                        '--implementation', 'cp',
                        '--python-version', '27',
                        '--abi', 'none',
                        '--disable-pip-version-check',
                        packages[0]])
                self._extract(tmp, vendor_dir)

            shutil.copyfile(tmpspec.name, spec)
            self.repository.add_remove_files(vendor_dir)

    def _update_packages(self, spec, packages):
        for package in packages:
            if not all(package.partition('==')):
                raise Exception('Package {} must be in the format name==version'.format(package))

        requirements = {}
        with open(spec, 'r') as f:
            for line in f.readlines():
                name, version = line.rstrip().split('==')
                requirements[name] = version
        for package in packages:
            name, version = package.split('==')
            requirements[name] = version

        with open(spec, 'w') as f:
            for name, version in sorted(requirements.items()):
                f.write('{}=={}\n'.format(name, version))

    def _extract(self, src, dest):
        """extract source distribution into vendor directory"""
        finder = FileFinder(src)
        for path, _ in finder.find('*'):
            base, ext = os.path.splitext(path)
            if ext == '.whl':
                # Wheels would extract into a directory with the name of the package, but
                # we want the platform signifiers, minus the version number.
                # Wheel filenames look like:
                # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}
                bits = base.split('-')

                # Remove the version number.
                bits.pop(1)
                target = os.path.join(dest, '-'.join(bits))
                mozfile.remove(target)  # remove existing version of vendored package
                os.mkdir(target)
                mozfile.extract(os.path.join(finder.base, path), target)
            else:
                # packages extract into package-version directory name and we strip the version
                tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
                target = os.path.join(dest, tld.rpartition('-')[0])
                mozfile.remove(target)  # remove existing version of vendored package
                mozfile.move(tld, target)