Bug 1490253 - Replace pipenv with pip-tools for vendoring packages and dependencies; r=ahal
☠☠ backed out by be3c71747eff ☠ ☠
authorDave Hunt <dhunt@mozilla.com>
Tue, 09 Oct 2018 21:32:35 +0000
changeset 498780 f92dcf5319ae2b9f1a305c0e155c463584c5fdf2
parent 498779 644d4b1846b0b0b8a4c8b31742f66c6d8f510079
child 498781 7e925d47de47a3c5775c6169c039e9240fa4f21b
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1490253
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1490253 - Replace pipenv with pip-tools for vendoring packages and dependencies; r=ahal Depends on D7869 Differential Revision: https://phabricator.services.mozilla.com/D7870
Pipfile
Pipfile.lock
build/virtualenv_packages.txt
python/mozbuild/mozbuild/vendor_python.py
third_party/python/requirements.in
third_party/python/requirements.txt
deleted file mode 100644
--- a/Pipfile
+++ /dev/null
@@ -1,20 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[dev-packages]
-
-[packages]
-attrs = "==18.1.0"
-blessings = "==1.7"
-jsmin = "==2.1.0"
-json-e = "==2.7.0"
-pip-tools = "==3.0.0"
-pipenv = "==2018.5.18"
-pytest = "==3.6.2"
-python-hglib = "==2.4"
-requests = "==2.9.1"
-six = "==1.10.0"
-virtualenv = "==15.2.0"
-voluptuous = "==0.11.5"
deleted file mode 100644
--- a/Pipfile.lock
+++ /dev/null
@@ -1,172 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "e756c316803705f9230eb8dd3b53fd9a9aa0a146c7387e3caffb668e0f7ea223"
-        },
-        "pipfile-spec": 6,
-        "requires": {},
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "atomicwrites": {
-            "hashes": [
-                "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585",
-                "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6"
-            ],
-            "version": "==1.1.5"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265",
-                "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b"
-            ],
-            "index": "pypi",
-            "version": "==18.1.0"
-        },
-        "blessings": {
-            "hashes": [
-                "sha256:98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d",
-                "sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3",
-                "sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e"
-            ],
-            "index": "pypi",
-            "version": "==1.7"
-        },
-        "certifi": {
-            "hashes": [
-                "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7",
-                "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0"
-            ],
-            "version": "==2018.4.16"
-        },
-        "click": {
-            "hashes": [
-                "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
-                "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
-            ],
-            "version": "==7.0"
-        },
-        "funcsigs": {
-            "hashes": [
-                "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca",
-                "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50"
-            ],
-            "markers": "python_version < '3.0'",
-            "version": "==1.0.2"
-        },
-        "jsmin": {
-            "hashes": [
-                "sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56"
-            ],
-            "index": "pypi",
-            "version": "==2.1.0"
-        },
-        "json-e": {
-            "hashes": [
-                "sha256:d8c1ec3f5bbc7728c3a504ebe58829f283c64eca230871e4eefe974b4cdaae4a"
-            ],
-            "index": "pypi",
-            "version": "==2.7.0"
-        },
-        "more-itertools": {
-            "hashes": [
-                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
-                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
-                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
-            ],
-            "version": "==4.3.0"
-        },
-        "pip-tools": {
-            "hashes": [
-                "sha256:4a94997602848f77ff02f660c0fcdfeaf316924ebb236c865f9742ce212aa6f9",
-                "sha256:e45e5198ce3799068642ebb0e7c9be5520bcff944c0186f79c1199a2759c970a"
-            ],
-            "index": "pypi",
-            "version": "==3.0.0"
-        },
-        "pipenv": {
-            "hashes": [
-                "sha256:04b9a8b02a3ff12a5502b335850cfdb192adcfd1d6bbdb7a7c47cae9ab9ddece",
-                "sha256:e96d5bfa6822a17b2200d455aa5f9002c14361c50df1b1e51921479d7c09e741"
-            ],
-            "index": "pypi",
-            "version": "==2018.5.18"
-        },
-        "pluggy": {
-            "hashes": [
-                "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff",
-                "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c",
-                "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5"
-            ],
-            "version": "==0.6.0"
-        },
-        "py": {
-            "hashes": [
-                "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7",
-                "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e"
-            ],
-            "version": "==1.5.4"
-        },
-        "pytest": {
-            "hashes": [
-                "sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c",
-                "sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd"
-            ],
-            "index": "pypi",
-            "version": "==3.6.2"
-        },
-        "python-hglib": {
-            "hashes": [
-                "sha256:693d6ed92a6566e78802c7a03c256cda33d08c63ad3f00fcfa11379b184b9462"
-            ],
-            "index": "pypi",
-            "version": "==2.4"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8",
-                "sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f"
-            ],
-            "index": "pypi",
-            "version": "==2.9.1"
-        },
-        "six": {
-            "hashes": [
-                "sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1",
-                "sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a"
-            ],
-            "index": "pypi",
-            "version": "==1.10.0"
-        },
-        "virtualenv": {
-            "hashes": [
-                "sha256:1d7e241b431e7afce47e77f8843a276f652699d1fa4f93b9d8ce0076fd7b0b54",
-                "sha256:e8e05d4714a1c51a2f5921e62f547fcb0f713ebbe959e0a7f585cc8bef71d11f"
-            ],
-            "index": "pypi",
-            "version": "==15.2.0"
-        },
-        "virtualenv-clone": {
-            "hashes": [
-                "sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7",
-                "sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8"
-            ],
-            "version": "==0.3.0"
-        },
-        "voluptuous": {
-            "hashes": [
-                "sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1",
-                "sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef"
-            ],
-            "index": "pypi",
-            "version": "==0.11.5"
-        }
-    },
-    "develop": {}
-}
--- a/build/virtualenv_packages.txt
+++ b/build/virtualenv_packages.txt
@@ -4,16 +4,17 @@ mozilla.pth:python/mozbuild
 mozilla.pth:python/mozlint
 mozilla.pth:python/mozrelease
 mozilla.pth:python/mozterm
 mozilla.pth:python/mozversioncontrol
 mozilla.pth:python/l10n
 mozilla.pth:third_party/python/atomicwrites
 mozilla.pth:third_party/python/attrs/src
 mozilla.pth:third_party/python/blessings
+mozilla.pth:third_party/python/Click
 mozilla.pth:third_party/python/compare-locales
 mozilla.pth:third_party/python/configobj
 mozilla.pth:third_party/python/cram
 mozilla.pth:third_party/python/dlmanager
 mozilla.pth:third_party/python/fluent
 mozilla.pth:third_party/python/funcsigs
 mozilla.pth:third_party/python/futures
 mozilla.pth:third_party/python/more-itertools
--- a/python/mozbuild/mozbuild/vendor_python.py
+++ b/python/mozbuild/mozbuild/vendor_python.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
+import shutil
 import subprocess
 
 import mozfile
 import mozpack.path as mozpath
 from mozbuild.base import MozbuildObject
 from mozfile import NamedTemporaryFile, TemporaryDirectory
 from mozpack.files import FileFinder
 
@@ -19,46 +20,69 @@ class VendorPython(MozbuildObject):
     def vendor(self, packages=None):
         self.populate_logger()
         self.log_manager.enable_unstructured()
 
         vendor_dir = mozpath.join(
             self.topsrcdir, os.path.join('third_party', 'python'))
 
         packages = packages or []
-        pipenv = self.ensure_pipenv()
 
-        for package in packages:
-            if not all(package.partition('==')):
-                raise Exception('Package {} must be in the format name==version'.format(package))
+        self._activate_virtualenv()
+        pip_compile = os.path.join(self.virtualenv_manager.bin_path, 'pip-compile')
+        if not os.path.exists(pip_compile):
+            path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party', 'python', 'pip-tools'))
+            self.virtualenv_manager.install_pip_package(path, vendored=True)
+        spec = os.path.join(vendor_dir, 'requirements.in')
+        requirements = os.path.join(vendor_dir, 'requirements.txt')
 
-        for package in packages:
-            subprocess.check_call(
-                [pipenv, 'install', package],
-                cwd=self.topsrcdir)
+        with NamedTemporaryFile('w') as tmpspec:
+            shutil.copyfile(spec, tmpspec.name)
+            self._update_packages(tmpspec.name, packages)
 
-        with NamedTemporaryFile('w') as requirements:
-            # determine the dependency graph and generate requirements.txt
-            subprocess.check_call(
-                [pipenv, 'lock', '--requirements'],
-                cwd=self.topsrcdir,
-                stdout=requirements)
+            # resolve the dependencies and update requirements.txt
+            subprocess.check_output([
+                pip_compile,
+                tmpspec.name,
+                '--no-header',
+                '--no-index',
+                '--output-file', requirements,
+                '--generate-hashes'])
 
             with TemporaryDirectory() as tmp:
                 # use requirements.txt to download archived source distributions of all packages
                 self.virtualenv_manager._run_pip([
                     'download',
-                    '-r', requirements.name,
+                    '-r', requirements,
                     '--no-deps',
                     '--dest', tmp,
                     '--no-binary', ':all:',
                     '--disable-pip-version-check'])
                 self._extract(tmp, vendor_dir)
 
-        self.repository.add_remove_files(vendor_dir)
+            shutil.copyfile(tmpspec.name, spec)
+            self.repository.add_remove_files(vendor_dir)
+
+    def _update_packages(self, spec, packages):
+        for package in packages:
+            if not all(package.partition('==')):
+                raise Exception('Package {} must be in the format name==version'.format(package))
+
+        requirements = {}
+        with open(spec, 'r') as f:
+            for line in f.readlines():
+                name, version = line.rstrip().split('==')
+                requirements[name] = version
+        for package in packages:
+            name, version = package.split('==')
+            requirements[name] = version
+
+        with open(spec, 'w') as f:
+            for name, version in sorted(requirements.items()):
+                f.write('{}=={}\n'.format(name, version))
 
     def _extract(self, src, dest):
         """extract source distribution into vendor directory"""
         finder = FileFinder(src)
         for path, _ in finder.find('*'):
             # packages extract into package-version directory name and we strip the version
             tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
             target = os.path.join(dest, tld.rpartition('-')[0])
new file mode 100644
--- /dev/null
+++ b/third_party/python/requirements.in
@@ -0,0 +1,12 @@
+attrs==18.1.0
+blessings==1.7
+jsmin==2.1.0
+json-e==2.7.0
+pip-tools==3.0.0
+pipenv==2018.5.18
+pytest==3.6.2
+python-hglib==2.4
+requests==2.9.1
+six==1.10.0
+virtualenv==15.2.0
+voluptuous==0.11.5
new file mode 100644
--- /dev/null
+++ b/third_party/python/requirements.txt
@@ -0,0 +1,68 @@
+atomicwrites==1.1.5 \
+    --hash=sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585 \
+    --hash=sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6 \
+    # via pytest
+attrs==18.1.0 \
+    --hash=sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265 \
+    --hash=sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b
+blessings==1.7 \
+    --hash=sha256:98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d \
+    --hash=sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3 \
+    --hash=sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e
+certifi==2018.4.16 \
+    --hash=sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7 \
+    --hash=sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0 \
+    # via pipenv
+click==7.0 \
+    --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
+    --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
+    # via pip-tools
+funcsigs==1.0.2 \
+    --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
+    --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 \
+    # via pytest
+jsmin==2.1.0 \
+    --hash=sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56
+json-e==2.7.0 \
+    --hash=sha256:d8c1ec3f5bbc7728c3a504ebe58829f283c64eca230871e4eefe974b4cdaae4a
+more-itertools==4.3.0 \
+    --hash=sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092 \
+    --hash=sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e \
+    --hash=sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d \
+    # via pytest
+pip-tools==3.0.0 \
+    --hash=sha256:4a94997602848f77ff02f660c0fcdfeaf316924ebb236c865f9742ce212aa6f9 \
+    --hash=sha256:e45e5198ce3799068642ebb0e7c9be5520bcff944c0186f79c1199a2759c970a
+pipenv==2018.5.18 \
+    --hash=sha256:04b9a8b02a3ff12a5502b335850cfdb192adcfd1d6bbdb7a7c47cae9ab9ddece \
+    --hash=sha256:e96d5bfa6822a17b2200d455aa5f9002c14361c50df1b1e51921479d7c09e741
+pluggy==0.6.0 \
+    --hash=sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff \
+    --hash=sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c \
+    --hash=sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5 \
+    # via pytest
+py==1.5.4 \
+    --hash=sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7 \
+    --hash=sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e \
+    # via pytest
+pytest==3.6.2 \
+    --hash=sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c \
+    --hash=sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd
+python-hglib==2.4 \
+    --hash=sha256:693d6ed92a6566e78802c7a03c256cda33d08c63ad3f00fcfa11379b184b9462
+requests==2.9.1 \
+    --hash=sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8 \
+    --hash=sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f
+six==1.10.0 \
+    --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \
+    --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a
+virtualenv-clone==0.3.0 \
+    --hash=sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7 \
+    --hash=sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8 \
+    # via pipenv
+virtualenv==15.2.0 \
+    --hash=sha256:1d7e241b431e7afce47e77f8843a276f652699d1fa4f93b9d8ce0076fd7b0b54 \
+    --hash=sha256:e8e05d4714a1c51a2f5921e62f547fcb0f713ebbe959e0a7f585cc8bef71d11f
+voluptuous==0.11.5 \
+    --hash=sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1 \
+    --hash=sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef