author | Dave Hunt <dhunt@mozilla.com> |
Wed, 10 Oct 2018 08:58:41 +0000 | |
changeset 440439 | 1fead446e3b9c18764a813099b7e6d932ac7d918 |
parent 440438 | 9a70e53741767b0c2e149e16f5a33594bebc89ba |
child 440440 | 41725614e772232c3ebcc7c2ead5330414f1a465 |
push id | 70667 |
push user | dhunt@mozilla.com |
push date | Wed, 10 Oct 2018 09:04:50 +0000 |
treeherder | autoland@1fead446e3b9 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ahal |
bugs | 1490253 |
milestone | 64.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
Pipfile | file | annotate | diff | comparison | revisions | |
Pipfile.lock | file | annotate | diff | comparison | revisions | |
build/virtualenv_packages.txt | file | annotate | diff | comparison | revisions | |
moz.build | file | annotate | diff | comparison | revisions | |
python/mozbuild/mozbuild/vendor_python.py | file | annotate | diff | comparison | revisions | |
third_party/python/moz.build | file | annotate | diff | comparison | revisions | |
third_party/python/requirements.in | file | annotate | diff | comparison | revisions | |
third_party/python/requirements.txt | file | annotate | diff | comparison | revisions |
deleted file mode 100644 --- a/Pipfile +++ /dev/null @@ -1,20 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[dev-packages] - -[packages] -attrs = "==18.1.0" -blessings = "==1.7" -jsmin = "==2.1.0" -json-e = "==2.7.0" -pip-tools = "==3.0.0" -pipenv = "==2018.5.18" -pytest = "==3.6.2" -python-hglib = "==2.4" -requests = "==2.9.1" -six = "==1.10.0" -virtualenv = "==15.2.0" -voluptuous = "==0.11.5"
deleted file mode 100644 --- a/Pipfile.lock +++ /dev/null @@ -1,172 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "e756c316803705f9230eb8dd3b53fd9a9aa0a146c7387e3caffb668e0f7ea223" - }, - "pipfile-spec": 6, - "requires": {}, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "atomicwrites": { - "hashes": [ - "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585", - "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6" - ], - "version": "==1.1.5" - }, - "attrs": { - "hashes": [ - "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265", - "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b" - ], - "index": "pypi", - "version": "==18.1.0" - }, - "blessings": { - "hashes": [ - "sha256:98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d", - "sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3", - "sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e" - ], - "index": "pypi", - "version": "==1.7" - }, - "certifi": { - "hashes": [ - "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", - "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0" - ], - "version": "==2018.4.16" - }, - "click": { - "hashes": [ - "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", - "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" - ], - "version": "==7.0" - }, - "funcsigs": { - "hashes": [ - "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", - "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" - ], - "markers": "python_version < '3.0'", - "version": "==1.0.2" - }, - "jsmin": { - "hashes": [ - "sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56" - ], - "index": "pypi", - "version": "==2.1.0" - }, - "json-e": { - "hashes": [ - "sha256:d8c1ec3f5bbc7728c3a504ebe58829f283c64eca230871e4eefe974b4cdaae4a" - ], - "index": "pypi", - "version": "==2.7.0" - }, - "more-itertools": { - "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" - ], - "version": "==4.3.0" - }, - "pip-tools": { - "hashes": [ - "sha256:4a94997602848f77ff02f660c0fcdfeaf316924ebb236c865f9742ce212aa6f9", - "sha256:e45e5198ce3799068642ebb0e7c9be5520bcff944c0186f79c1199a2759c970a" - ], - "index": "pypi", - "version": "==3.0.0" - }, - "pipenv": { - "hashes": [ - "sha256:04b9a8b02a3ff12a5502b335850cfdb192adcfd1d6bbdb7a7c47cae9ab9ddece", - "sha256:e96d5bfa6822a17b2200d455aa5f9002c14361c50df1b1e51921479d7c09e741" - ], - "index": "pypi", - "version": "==2018.5.18" - }, - "pluggy": { - "hashes": [ - "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff", - "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c", - "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5" - ], - "version": "==0.6.0" - }, - "py": { - "hashes": [ - "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7", - "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e" - ], - "version": "==1.5.4" - }, - "pytest": { - "hashes": [ - "sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c", - "sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd" - ], - "index": "pypi", - "version": "==3.6.2" - }, - "python-hglib": { - "hashes": [ - "sha256:693d6ed92a6566e78802c7a03c256cda33d08c63ad3f00fcfa11379b184b9462" - ], - "index": "pypi", - "version": "==2.4" - }, - "requests": { - "hashes": [ - "sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8", - "sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f" - ], - "index": "pypi", - "version": "==2.9.1" - }, - "six": { - "hashes": [ - "sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1", - "sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a" - ], - "index": "pypi", - "version": "==1.10.0" - }, - "virtualenv": { - "hashes": [ - "sha256:1d7e241b431e7afce47e77f8843a276f652699d1fa4f93b9d8ce0076fd7b0b54", - "sha256:e8e05d4714a1c51a2f5921e62f547fcb0f713ebbe959e0a7f585cc8bef71d11f" - ], - "index": "pypi", - "version": "==15.2.0" - }, - "virtualenv-clone": { - "hashes": [ - "sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7", - "sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8" - ], - "version": "==0.3.0" - }, - "voluptuous": { - "hashes": [ - "sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1", - "sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef" - ], - "index": "pypi", - "version": "==0.11.5" - } - }, - "develop": {} -}
--- a/build/virtualenv_packages.txt +++ b/build/virtualenv_packages.txt @@ -4,16 +4,17 @@ mozilla.pth:python/mozbuild mozilla.pth:python/mozlint mozilla.pth:python/mozrelease mozilla.pth:python/mozterm mozilla.pth:python/mozversioncontrol mozilla.pth:python/l10n mozilla.pth:third_party/python/atomicwrites mozilla.pth:third_party/python/attrs/src mozilla.pth:third_party/python/blessings +mozilla.pth:third_party/python/Click mozilla.pth:third_party/python/compare-locales mozilla.pth:third_party/python/configobj mozilla.pth:third_party/python/cram mozilla.pth:third_party/python/dlmanager mozilla.pth:third_party/python/fluent mozilla.pth:third_party/python/funcsigs mozilla.pth:third_party/python/futures mozilla.pth:third_party/python/more-itertools
--- a/moz.build +++ b/moz.build @@ -32,19 +32,16 @@ with Files('mach'): BUG_COMPONENT = ('Core', 'mach') with Files('*moz*'): BUG_COMPONENT = ('Firefox Build System', 'General') with Files('GNUmakefile'): BUG_COMPONENT = ('Firefox Build System', 'General') -with Files('Pipfile*'): - BUG_COMPONENT = ('Firefox Build System', 'General') - with Files('*gradle*'): BUG_COMPONENT = ('Firefox for Android', 'Build Config & IDE Support') SCHEDULES.exclusive = ['android'] with Files('*.json'): BUG_COMPONENT = ('Firefox Build System', 'General') with Files('**/l10n.toml'):
--- a/python/mozbuild/mozbuild/vendor_python.py +++ b/python/mozbuild/mozbuild/vendor_python.py @@ -1,15 +1,16 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, print_function, unicode_literals import os +import shutil import subprocess import mozfile import mozpack.path as mozpath from mozbuild.base import MozbuildObject from mozfile import NamedTemporaryFile, TemporaryDirectory from mozpack.files import FileFinder @@ -19,46 +20,69 @@ class VendorPython(MozbuildObject): def vendor(self, packages=None): self.populate_logger() self.log_manager.enable_unstructured() vendor_dir = mozpath.join( self.topsrcdir, os.path.join('third_party', 'python')) packages = packages or [] - pipenv = self.ensure_pipenv() - for package in packages: - if not all(package.partition('==')): - raise Exception('Package {} must be in the format name==version'.format(package)) + self._activate_virtualenv() + pip_compile = os.path.join(self.virtualenv_manager.bin_path, 'pip-compile') + if not os.path.exists(pip_compile): + path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party', 'python', 'pip-tools')) + self.virtualenv_manager.install_pip_package(path, vendored=True) + spec = os.path.join(vendor_dir, 'requirements.in') + requirements = os.path.join(vendor_dir, 'requirements.txt') - for package in packages: - subprocess.check_call( - [pipenv, 'install', package], - cwd=self.topsrcdir) + with NamedTemporaryFile('w') as tmpspec: + shutil.copyfile(spec, tmpspec.name) + self._update_packages(tmpspec.name, packages) - with NamedTemporaryFile('w') as requirements: - # determine the dependency graph and generate requirements.txt - subprocess.check_call( - [pipenv, 'lock', '--requirements'], - cwd=self.topsrcdir, - stdout=requirements) + # resolve the dependencies and update requirements.txt + subprocess.check_output([ + pip_compile, + tmpspec.name, + '--no-header', + '--no-index', + '--output-file', requirements, + '--generate-hashes']) with TemporaryDirectory() as tmp: # use requirements.txt to download archived source distributions of all packages self.virtualenv_manager._run_pip([ 'download', - '-r', requirements.name, + '-r', requirements, '--no-deps', '--dest', tmp, '--no-binary', ':all:', '--disable-pip-version-check']) self._extract(tmp, vendor_dir) - self.repository.add_remove_files(vendor_dir) + shutil.copyfile(tmpspec.name, spec) + self.repository.add_remove_files(vendor_dir) + + def _update_packages(self, spec, packages): + for package in packages: + if not all(package.partition('==')): + raise Exception('Package {} must be in the format name==version'.format(package)) + + requirements = {} + with open(spec, 'r') as f: + for line in f.readlines(): + name, version = line.rstrip().split('==') + requirements[name] = version + for package in packages: + name, version = package.split('==') + requirements[name] = version + + with open(spec, 'w') as f: + for name, version in sorted(requirements.items()): + f.write('{}=={}\n'.format(name, version)) def _extract(self, src, dest): """extract source distribution into vendor directory""" finder = FileFinder(src) for path, _ in finder.find('*'): # packages extract into package-version directory name and we strip the version tld = mozfile.extract(os.path.join(finder.base, path), dest)[0] target = os.path.join(dest, tld.rpartition('-')[0])
--- a/third_party/python/moz.build +++ b/third_party/python/moz.build @@ -69,16 +69,19 @@ with Files('pyyaml/**'): BUG_COMPONENT = ('Taskcluster', 'General') with Files('redo/**'): BUG_COMPONENT = ('Firefox Build System', 'General') with Files('requests*/**'): BUG_COMPONENT = ('Firefox Build System', 'General') +with Files('requirements.*'): + BUG_COMPONENT = ('Firefox Build System', 'General') + with Files('rsa/**'): BUG_COMPONENT = ('Core', 'Security: PSM') with Files('slugid/**'): BUG_COMPONENT = ('Taskcluster', 'Platform Libraries') with Files('virtualenv/**'): BUG_COMPONENT = ('Firefox Build System', 'General')
new file mode 100644 --- /dev/null +++ b/third_party/python/requirements.in @@ -0,0 +1,12 @@ +attrs==18.1.0 +blessings==1.7 +jsmin==2.1.0 +json-e==2.7.0 +pip-tools==3.0.0 +pipenv==2018.5.18 +pytest==3.6.2 +python-hglib==2.4 +requests==2.9.1 +six==1.10.0 +virtualenv==15.2.0 +voluptuous==0.11.5
new file mode 100644 --- /dev/null +++ b/third_party/python/requirements.txt @@ -0,0 +1,68 @@ +atomicwrites==1.1.5 \ + --hash=sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585 \ + --hash=sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6 \ + # via pytest +attrs==18.1.0 \ + --hash=sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265 \ + --hash=sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b +blessings==1.7 \ + --hash=sha256:98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d \ + --hash=sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3 \ + --hash=sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e +certifi==2018.4.16 \ + --hash=sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7 \ + --hash=sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0 \ + # via pipenv +click==7.0 \ + --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ + --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ + # via pip-tools +funcsigs==1.0.2 \ + --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ + --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 \ + # via pytest +jsmin==2.1.0 \ + --hash=sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56 +json-e==2.7.0 \ + --hash=sha256:d8c1ec3f5bbc7728c3a504ebe58829f283c64eca230871e4eefe974b4cdaae4a +more-itertools==4.3.0 \ + --hash=sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092 \ + --hash=sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e \ + --hash=sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d \ + # via pytest +pip-tools==3.0.0 \ + --hash=sha256:4a94997602848f77ff02f660c0fcdfeaf316924ebb236c865f9742ce212aa6f9 \ + --hash=sha256:e45e5198ce3799068642ebb0e7c9be5520bcff944c0186f79c1199a2759c970a +pipenv==2018.5.18 \ + --hash=sha256:04b9a8b02a3ff12a5502b335850cfdb192adcfd1d6bbdb7a7c47cae9ab9ddece \ + --hash=sha256:e96d5bfa6822a17b2200d455aa5f9002c14361c50df1b1e51921479d7c09e741 +pluggy==0.6.0 \ + --hash=sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff \ + --hash=sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c \ + --hash=sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5 \ + # via pytest +py==1.5.4 \ + --hash=sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7 \ + --hash=sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e \ + # via pytest +pytest==3.6.2 \ + --hash=sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c \ + --hash=sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd +python-hglib==2.4 \ + --hash=sha256:693d6ed92a6566e78802c7a03c256cda33d08c63ad3f00fcfa11379b184b9462 +requests==2.9.1 \ + --hash=sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8 \ + --hash=sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f +six==1.10.0 \ + --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ + --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a +virtualenv-clone==0.3.0 \ + --hash=sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7 \ + --hash=sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8 \ + # via pipenv +virtualenv==15.2.0 \ + --hash=sha256:1d7e241b431e7afce47e77f8843a276f652699d1fa4f93b9d8ce0076fd7b0b54 \ + --hash=sha256:e8e05d4714a1c51a2f5921e62f547fcb0f713ebbe959e0a7f585cc8bef71d11f +voluptuous==0.11.5 \ + --hash=sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1 \ + --hash=sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef