Bug 1346026 - Add ability to vendor Python modules using mach; r?ahal, ted draft
authorDave Hunt <dhunt@mozilla.com>
Wed, 09 May 2018 16:11:40 +0100
changeset 798854 b1831ecca3785c62a3e69aeac8667ed632fa1976
parent 793056 9294f67b3f3bd4a3dd898961148cecd8bfc1ce9c
child 798855 37725327e69198362a3e1ce947d3bbc67c2ab639
child 798925 726601dbf354cfbf42676fbe099a7b710fc43211
push id110856
push userbmo:dave.hunt@gmail.com
push dateWed, 23 May 2018 15:36:53 +0000
reviewersahal, ted
bugs1346026
milestone62.0a1
Bug 1346026 - Add ability to vendor Python modules using mach; r?ahal, ted To vendor a Python package, run ``mach vendor python [PACKAGE]``, where ``[PACKAGE]`` is one or more package names along with a version number in the format ``pytest==3.5.1``. The package will be installed, transient dependencies will be determined, and a ``requirements.txt`` file will be generated with the full list of dependencies. The requirements file is then used with ``pip`` to download and extract the source distributions of all packages into the ``third_party/python`` directory. If you're familiar with ``Pipfile`` you can also directly modify this in the in the top source directory and then run ``mach vendor python`` for your changes to take effect. This allows advanced options such as specifying alternative package indexed (see below), and `PEP 508 specifiers <https://www.python.org/dev/peps/pep-0508/>`_.ยง MozReview-Commit-ID: CRWoFamUy7V
python/docs/index.rst
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/mach_commands.py
python/mozbuild/mozbuild/vendor_python.py
--- a/python/docs/index.rst
+++ b/python/docs/index.rst
@@ -21,16 +21,49 @@ maintainer removing an old tarball to de
 
 Where possible, the following policy applies to **ALL** vendored packages:
 
 * Vendored libraries **SHOULD NOT** be modified except as required to
   successfully vendor them.
 * Vendored libraries **SHOULD** be released copies of libraries available on
   PyPI.
 
+
+Adding a Python package
+~~~~~~~~~~~~~~~~~~~~~~~
+
+To vendor a Python package, run ``mach vendor python [PACKAGE]``, where
+``[PACKAGE]`` is one or more package names along with a version number in the
+format ``pytest==3.5.1``. The package will be installed, transient dependencies
+will be determined, and a ``requirements.txt`` file will be generated with the
+full list of dependencies. The requirements file is then used with ``pip`` to
+download and extract the source distributions of all packages into the
+``third_party/python`` directory.
+
+If you're familiar with ``Pipfile`` you can also directly modify this in the in
+the top source directory and then run ``mach vendor python`` for your changes
+to take effect. This allows advanced options such as specifying alternative
+package indexes (see below), and
+`PEP 508 specifiers <https://www.python.org/dev/peps/pep-0508/>`_.
+
+Note that the `specification <https://github.com/pypa/pipfile>`_ for
+``Pipfile`` and ``Pipfile.lock`` is still in active development. More
+information can be found in the
+`Pipenv documentation <https://docs.pipenv.org/>`_, which is the reference
+implementation we're using.
+
+What if the package isn't on PyPI?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the package is available on another Python package index, then you can add
+these details to ``Pipfile`` by
+`specifying package indexes <https://docs.pipenv.org/advanced/#specifying-package-indexes>`_.
+If the package isn't available on any Python package index, then you can
+manually copy the source distribution into the ``third_party/python`` directory.
+
 Using a Python package index
 ============================
 
 If the Python package is not used in the building of Firefox then it can be
 installed from a package index. Some tasks are not permitted to use external
 resources, and for those we can publish packages to an internal PyPI mirror.
 See `how to upload to internal PyPI <https://wiki.mozilla.org/ReleaseEngineering/How_To/Upload_to_internal_Pypi>`_
 for more details. If you are not restricted, you can install packages from PyPI
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -743,22 +743,29 @@ class MozbuildObject(ProcessExecutionMix
     def _activate_virtualenv(self):
         self.virtualenv_manager.ensure()
         self.virtualenv_manager.activate()
 
 
     def _set_log_level(self, verbose):
         self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG)
 
+    def ensure_pipenv(self):
+        self._activate_virtualenv()
+        pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv')
+        if not os.path.exists(pipenv):
+            pipenv_reqs = os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/pipenv.txt')
+            self.virtualenv_manager.install_pip_requirements(
+                pipenv_reqs, require_hashes=False, vendored=True)
+        return pipenv
+
     def activate_pipenv(self, path):
         if not os.path.exists(path):
             raise Exception('Pipfile not found: %s.' % path)
-        self._activate_virtualenv()
-        pipenv_reqs = os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/pipenv.txt')
-        self.virtualenv_manager.install_pip_requirements(pipenv_reqs, require_hashes=False, vendored=True)
+        self.ensure_pipenv()
         self.virtualenv_manager.activate_pipenv(path)
 
 
 class MachCommandBase(MozbuildObject):
     """Base class for mach command providers that wish to be MozbuildObjects.
 
     This provides a level of indirection so MozbuildObject can be refactored
     without having to change everything that inherits from it.
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -2110,16 +2110,24 @@ class Vendor(MachCommandBase):
     @CommandArgument('--ignore-modified', action='store_true',
         help='Ignore modified files in current checkout',
         default=False)
     def vendor_aom(self, **kwargs):
         from mozbuild.vendor_aom import VendorAOM
         vendor_command = self._spawn(VendorAOM)
         vendor_command.vendor(**kwargs)
 
+    @SubCommand('vendor', 'python',
+                description='Vendor Python packages from pypi.org into third_party/python')
+    @CommandArgument('packages', default=None, nargs='*', help='Packages to vendor. If omitted, packages and their dependencies defined in Pipfile.lock will be vendored. If Pipfile has been modified, then Pipfile.lock will be regenerated. Note that transient dependencies may be updated when running this command.')
+    def vendor_python(self, **kwargs):
+        from mozbuild.vendor_python import VendorPython
+        vendor_command = self._spawn(VendorPython)
+        vendor_command.vendor(**kwargs)
+
 
 @CommandProvider
 class WebRTCGTestCommands(GTestCommands):
     @Command('webrtc-gtest', category='testing',
         description='Run WebRTC.org GTest unit tests.')
     @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter',
         help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns),"
              "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).")
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/vendor_python.py
@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import subprocess
+
+import mozfile
+import mozpack.path as mozpath
+from mozbuild.base import MozbuildObject
+from mozfile import NamedTemporaryFile, TemporaryDirectory
+from mozpack.files import FileFinder
+
+
+class VendorPython(MozbuildObject):
+
+    def vendor(self, packages=None):
+        self.populate_logger()
+        self.log_manager.enable_unstructured()
+
+        vendor_dir = mozpath.join(
+            self.topsrcdir, os.path.join('third_party', 'python'))
+
+        packages = packages or []
+        pipenv = self.ensure_pipenv()
+
+        for package in packages:
+            if not all(package.partition('==')):
+                raise Exception('Package {} must be in the format name==version'.format(package))
+
+        for package in packages:
+            subprocess.check_call(
+                [pipenv, 'install', package],
+                cwd=self.topsrcdir)
+
+        with NamedTemporaryFile('w') as requirements:
+            # determine the dependency graph and generate requirements.txt
+            subprocess.check_call(
+                [pipenv, 'lock', '--requirements'],
+                cwd=self.topsrcdir,
+                stdout=requirements)
+
+            with TemporaryDirectory() as tmp:
+                # use requirements.txt to download archived source distributions of all packages
+                self.virtualenv_manager._run_pip([
+                    'download',
+                    '-r', requirements.name,
+                    '--no-deps',
+                    '--dest', tmp,
+                    '--no-binary', ':all:',
+                    '--disable-pip-version-check'])
+                self._extract(tmp, vendor_dir)
+
+        self.repository.add_remove_files(vendor_dir)
+
+    def _extract(self, src, dest):
+        """extract source distribution into vendor directory"""
+        finder = FileFinder(src)
+        for path, _ in finder.find('*'):
+            # packages extract into package-version directory name and we strip the version
+            tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
+            target = os.path.join(dest, tld.rpartition('-')[0])
+            mozfile.remove(target)  # remove existing version of vendored package
+            mozfile.move(tld, target)