Bug 1490253 - Replace pipenv with pip-tools for vendoring packages and dependencies; r=ahal
authorDave Hunt <dhunt@mozilla.com>
Wed, 10 Oct 2018 08:58:41 +0000
changeset 440425 1fead446e3b9c18764a813099b7e6d932ac7d918
parent 440424 9a70e53741767b0c2e149e16f5a33594bebc89ba
child 440426 41725614e772232c3ebcc7c2ead5330414f1a465
push id34820
push userdvarga@mozilla.com
push dateWed, 10 Oct 2018 16:07:50 +0000
treeherdermozilla-central@0f1d5395f801 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1490253
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1490253 - Replace pipenv with pip-tools for vendoring packages and dependencies; r=ahal Depends on D7869 Differential Revision: https://phabricator.services.mozilla.com/D7870
Pipfile
Pipfile.lock
build/virtualenv_packages.txt
moz.build
python/mozbuild/mozbuild/vendor_python.py
third_party/python/moz.build
third_party/python/requirements.in
third_party/python/requirements.txt
deleted file mode 100644
--- a/Pipfile
+++ /dev/null
@@ -1,20 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[dev-packages]
-
-[packages]
-attrs = "==18.1.0"
-blessings = "==1.7"
-jsmin = "==2.1.0"
-json-e = "==2.7.0"
-pip-tools = "==3.0.0"
-pipenv = "==2018.5.18"
-pytest = "==3.6.2"
-python-hglib = "==2.4"
-requests = "==2.9.1"
-six = "==1.10.0"
-virtualenv = "==15.2.0"
-voluptuous = "==0.11.5"
deleted file mode 100644
--- a/Pipfile.lock
+++ /dev/null
@@ -1,172 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "e756c316803705f9230eb8dd3b53fd9a9aa0a146c7387e3caffb668e0f7ea223"
-        },
-        "pipfile-spec": 6,
-        "requires": {},
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "atomicwrites": {
-            "hashes": [
-                "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585",
-                "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6"
-            ],
-            "version": "==1.1.5"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265",
-                "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b"
-            ],
-            "index": "pypi",
-            "version": "==18.1.0"
-        },
-        "blessings": {
-            "hashes": [
-                "sha256:98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d",
-                "sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3",
-                "sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e"
-            ],
-            "index": "pypi",
-            "version": "==1.7"
-        },
-        "certifi": {
-            "hashes": [
-                "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7",
-                "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0"
-            ],
-            "version": "==2018.4.16"
-        },
-        "click": {
-            "hashes": [
-                "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
-                "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
-            ],
-            "version": "==7.0"
-        },
-        "funcsigs": {
-            "hashes": [
-                "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca",
-                "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50"
-            ],
-            "markers": "python_version < '3.0'",
-            "version": "==1.0.2"
-        },
-        "jsmin": {
-            "hashes": [
-                "sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56"
-            ],
-            "index": "pypi",
-            "version": "==2.1.0"
-        },
-        "json-e": {
-            "hashes": [
-                "sha256:d8c1ec3f5bbc7728c3a504ebe58829f283c64eca230871e4eefe974b4cdaae4a"
-            ],
-            "index": "pypi",
-            "version": "==2.7.0"
-        },
-        "more-itertools": {
-            "hashes": [
-                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
-                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
-                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
-            ],
-            "version": "==4.3.0"
-        },
-        "pip-tools": {
-            "hashes": [
-                "sha256:4a94997602848f77ff02f660c0fcdfeaf316924ebb236c865f9742ce212aa6f9",
-                "sha256:e45e5198ce3799068642ebb0e7c9be5520bcff944c0186f79c1199a2759c970a"
-            ],
-            "index": "pypi",
-            "version": "==3.0.0"
-        },
-        "pipenv": {
-            "hashes": [
-                "sha256:04b9a8b02a3ff12a5502b335850cfdb192adcfd1d6bbdb7a7c47cae9ab9ddece",
-                "sha256:e96d5bfa6822a17b2200d455aa5f9002c14361c50df1b1e51921479d7c09e741"
-            ],
-            "index": "pypi",
-            "version": "==2018.5.18"
-        },
-        "pluggy": {
-            "hashes": [
-                "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff",
-                "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c",
-                "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5"
-            ],
-            "version": "==0.6.0"
-        },
-        "py": {
-            "hashes": [
-                "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7",
-                "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e"
-            ],
-            "version": "==1.5.4"
-        },
-        "pytest": {
-            "hashes": [
-                "sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c",
-                "sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd"
-            ],
-            "index": "pypi",
-            "version": "==3.6.2"
-        },
-        "python-hglib": {
-            "hashes": [
-                "sha256:693d6ed92a6566e78802c7a03c256cda33d08c63ad3f00fcfa11379b184b9462"
-            ],
-            "index": "pypi",
-            "version": "==2.4"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8",
-                "sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f"
-            ],
-            "index": "pypi",
-            "version": "==2.9.1"
-        },
-        "six": {
-            "hashes": [
-                "sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1",
-                "sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a"
-            ],
-            "index": "pypi",
-            "version": "==1.10.0"
-        },
-        "virtualenv": {
-            "hashes": [
-                "sha256:1d7e241b431e7afce47e77f8843a276f652699d1fa4f93b9d8ce0076fd7b0b54",
-                "sha256:e8e05d4714a1c51a2f5921e62f547fcb0f713ebbe959e0a7f585cc8bef71d11f"
-            ],
-            "index": "pypi",
-            "version": "==15.2.0"
-        },
-        "virtualenv-clone": {
-            "hashes": [
-                "sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7",
-                "sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8"
-            ],
-            "version": "==0.3.0"
-        },
-        "voluptuous": {
-            "hashes": [
-                "sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1",
-                "sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef"
-            ],
-            "index": "pypi",
-            "version": "==0.11.5"
-        }
-    },
-    "develop": {}
-}
--- a/build/virtualenv_packages.txt
+++ b/build/virtualenv_packages.txt
@@ -4,16 +4,17 @@ mozilla.pth:python/mozbuild
 mozilla.pth:python/mozlint
 mozilla.pth:python/mozrelease
 mozilla.pth:python/mozterm
 mozilla.pth:python/mozversioncontrol
 mozilla.pth:python/l10n
 mozilla.pth:third_party/python/atomicwrites
 mozilla.pth:third_party/python/attrs/src
 mozilla.pth:third_party/python/blessings
+mozilla.pth:third_party/python/Click
 mozilla.pth:third_party/python/compare-locales
 mozilla.pth:third_party/python/configobj
 mozilla.pth:third_party/python/cram
 mozilla.pth:third_party/python/dlmanager
 mozilla.pth:third_party/python/fluent
 mozilla.pth:third_party/python/funcsigs
 mozilla.pth:third_party/python/futures
 mozilla.pth:third_party/python/more-itertools
--- a/moz.build
+++ b/moz.build
@@ -32,19 +32,16 @@ with Files('mach'):
     BUG_COMPONENT = ('Core', 'mach')
 
 with Files('*moz*'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
 
 with Files('GNUmakefile'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
 
-with Files('Pipfile*'):
-    BUG_COMPONENT = ('Firefox Build System', 'General')
-
 with Files('*gradle*'):
     BUG_COMPONENT = ('Firefox for Android', 'Build Config & IDE Support')
     SCHEDULES.exclusive = ['android']
 
 with Files('*.json'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
 
 with Files('**/l10n.toml'):
--- a/python/mozbuild/mozbuild/vendor_python.py
+++ b/python/mozbuild/mozbuild/vendor_python.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
+import shutil
 import subprocess
 
 import mozfile
 import mozpack.path as mozpath
 from mozbuild.base import MozbuildObject
 from mozfile import NamedTemporaryFile, TemporaryDirectory
 from mozpack.files import FileFinder
 
@@ -19,46 +20,69 @@ class VendorPython(MozbuildObject):
     def vendor(self, packages=None):
         self.populate_logger()
         self.log_manager.enable_unstructured()
 
         vendor_dir = mozpath.join(
             self.topsrcdir, os.path.join('third_party', 'python'))
 
         packages = packages or []
-        pipenv = self.ensure_pipenv()
 
-        for package in packages:
-            if not all(package.partition('==')):
-                raise Exception('Package {} must be in the format name==version'.format(package))
+        self._activate_virtualenv()
+        pip_compile = os.path.join(self.virtualenv_manager.bin_path, 'pip-compile')
+        if not os.path.exists(pip_compile):
+            path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party', 'python', 'pip-tools'))
+            self.virtualenv_manager.install_pip_package(path, vendored=True)
+        spec = os.path.join(vendor_dir, 'requirements.in')
+        requirements = os.path.join(vendor_dir, 'requirements.txt')
 
-        for package in packages:
-            subprocess.check_call(
-                [pipenv, 'install', package],
-                cwd=self.topsrcdir)
+        with NamedTemporaryFile('w') as tmpspec:
+            shutil.copyfile(spec, tmpspec.name)
+            self._update_packages(tmpspec.name, packages)
 
-        with NamedTemporaryFile('w') as requirements:
-            # determine the dependency graph and generate requirements.txt
-            subprocess.check_call(
-                [pipenv, 'lock', '--requirements'],
-                cwd=self.topsrcdir,
-                stdout=requirements)
+            # resolve the dependencies and update requirements.txt
+            subprocess.check_output([
+                pip_compile,
+                tmpspec.name,
+                '--no-header',
+                '--no-index',
+                '--output-file', requirements,
+                '--generate-hashes'])
 
             with TemporaryDirectory() as tmp:
                 # use requirements.txt to download archived source distributions of all packages
                 self.virtualenv_manager._run_pip([
                     'download',
-                    '-r', requirements.name,
+                    '-r', requirements,
                     '--no-deps',
                     '--dest', tmp,
                     '--no-binary', ':all:',
                     '--disable-pip-version-check'])
                 self._extract(tmp, vendor_dir)
 
-        self.repository.add_remove_files(vendor_dir)
+            shutil.copyfile(tmpspec.name, spec)
+            self.repository.add_remove_files(vendor_dir)
+
+    def _update_packages(self, spec, packages):
+        for package in packages:
+            if not all(package.partition('==')):
+                raise Exception('Package {} must be in the format name==version'.format(package))
+
+        requirements = {}
+        with open(spec, 'r') as f:
+            for line in f.readlines():
+                name, version = line.rstrip().split('==')
+                requirements[name] = version
+        for package in packages:
+            name, version = package.split('==')
+            requirements[name] = version
+
+        with open(spec, 'w') as f:
+            for name, version in sorted(requirements.items()):
+                f.write('{}=={}\n'.format(name, version))
 
     def _extract(self, src, dest):
         """extract source distribution into vendor directory"""
         finder = FileFinder(src)
         for path, _ in finder.find('*'):
             # packages extract into package-version directory name and we strip the version
             tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
             target = os.path.join(dest, tld.rpartition('-')[0])
--- a/third_party/python/moz.build
+++ b/third_party/python/moz.build
@@ -69,16 +69,19 @@ with Files('pyyaml/**'):
     BUG_COMPONENT = ('Taskcluster', 'General')
 
 with Files('redo/**'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
 
 with Files('requests*/**'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
 
+with Files('requirements.*'):
+    BUG_COMPONENT = ('Firefox Build System', 'General')
+
 with Files('rsa/**'):
     BUG_COMPONENT = ('Core', 'Security: PSM')
 
 with Files('slugid/**'):
     BUG_COMPONENT = ('Taskcluster', 'Platform Libraries')
 
 with Files('virtualenv/**'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
new file mode 100644
--- /dev/null
+++ b/third_party/python/requirements.in
@@ -0,0 +1,12 @@
+attrs==18.1.0
+blessings==1.7
+jsmin==2.1.0
+json-e==2.7.0
+pip-tools==3.0.0
+pipenv==2018.5.18
+pytest==3.6.2
+python-hglib==2.4
+requests==2.9.1
+six==1.10.0
+virtualenv==15.2.0
+voluptuous==0.11.5
new file mode 100644
--- /dev/null
+++ b/third_party/python/requirements.txt
@@ -0,0 +1,68 @@
+atomicwrites==1.1.5 \
+    --hash=sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585 \
+    --hash=sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6 \
+    # via pytest
+attrs==18.1.0 \
+    --hash=sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265 \
+    --hash=sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b
+blessings==1.7 \
+    --hash=sha256:98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d \
+    --hash=sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3 \
+    --hash=sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e
+certifi==2018.4.16 \
+    --hash=sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7 \
+    --hash=sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0 \
+    # via pipenv
+click==7.0 \
+    --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
+    --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
+    # via pip-tools
+funcsigs==1.0.2 \
+    --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
+    --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 \
+    # via pytest
+jsmin==2.1.0 \
+    --hash=sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56
+json-e==2.7.0 \
+    --hash=sha256:d8c1ec3f5bbc7728c3a504ebe58829f283c64eca230871e4eefe974b4cdaae4a
+more-itertools==4.3.0 \
+    --hash=sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092 \
+    --hash=sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e \
+    --hash=sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d \
+    # via pytest
+pip-tools==3.0.0 \
+    --hash=sha256:4a94997602848f77ff02f660c0fcdfeaf316924ebb236c865f9742ce212aa6f9 \
+    --hash=sha256:e45e5198ce3799068642ebb0e7c9be5520bcff944c0186f79c1199a2759c970a
+pipenv==2018.5.18 \
+    --hash=sha256:04b9a8b02a3ff12a5502b335850cfdb192adcfd1d6bbdb7a7c47cae9ab9ddece \
+    --hash=sha256:e96d5bfa6822a17b2200d455aa5f9002c14361c50df1b1e51921479d7c09e741
+pluggy==0.6.0 \
+    --hash=sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff \
+    --hash=sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c \
+    --hash=sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5 \
+    # via pytest
+py==1.5.4 \
+    --hash=sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7 \
+    --hash=sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e \
+    # via pytest
+pytest==3.6.2 \
+    --hash=sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c \
+    --hash=sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd
+python-hglib==2.4 \
+    --hash=sha256:693d6ed92a6566e78802c7a03c256cda33d08c63ad3f00fcfa11379b184b9462
+requests==2.9.1 \
+    --hash=sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8 \
+    --hash=sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f
+six==1.10.0 \
+    --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \
+    --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a
+virtualenv-clone==0.3.0 \
+    --hash=sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7 \
+    --hash=sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8 \
+    # via pipenv
+virtualenv==15.2.0 \
+    --hash=sha256:1d7e241b431e7afce47e77f8843a276f652699d1fa4f93b9d8ce0076fd7b0b54 \
+    --hash=sha256:e8e05d4714a1c51a2f5921e62f547fcb0f713ebbe959e0a7f585cc8bef71d11f
+voluptuous==0.11.5 \
+    --hash=sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1 \
+    --hash=sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef