Bug 1356101 - Derive the mach_bootstrap search path from build/virtualenv_packages.txt. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 14 Apr 2017 08:19:08 +0900
changeset 353233 54f9c9d1a0948d615d4105d753568bc502bae52b
parent 353232 98c08360e61b83fe1eeefd42c93a9099ebf86806
child 353234 b17ddbfb0f551c6b4d24c6d2049f5405713697e7
push id31660
push userarchaeopteryx@coole-files.de
push dateSat, 15 Apr 2017 17:57:59 +0000
treeherdermozilla-central@ed88aa504601 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1356101, 893976, 1244736
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1356101 - Derive the mach_bootstrap search path from build/virtualenv_packages.txt. r=gps Most entries in virtualenv_packages.txt that are .pth or packages.txt are currently in SEARCH_PATHS in mach_bootstrap. The ones that are missing would make sense in SEARCH_PATHS. None of non-.pth or packages.txt entries, however, are in SEARCH_PATHS and don't make sense there. On the other hand, virtualenv_packages.txt misses a lot of things that are in SEARCH_PATHS, all of which should be there. One exception: xpcom/idl-parser, which causes problems due to the xpidl package containing an xpidl module, which causes problems with the in-tree scripts using it. Plus, it needs a cache directory, which is messy, so it's preferable to keep it away from the virtualenv. It turns out it was added to mach_bootstrap.py in bug 893976 for a command that was since then removed (bug 1244736), so we can get away with removing it. So instead of keeping those two separate lists out of sync, we replace the SEARCH_PATHS list from mach_bootstrap with one that is derived at runtime from the contents of virtualenv_packages.txt. And since a .pth can't fail to install in the virtualenv, it makes no sense to have psutil.pth defined as optional, which allows it to end up in the mach_bootstrap search path automatically. Finally, because we do have overlapping module names in the tree (e.g. runtests), and mach_bootstrap's SEARCH_PATHS had a guaranteed order, we change the order of the virtualenv_packages.txt file to match what used to be in mach_bootstrap, and make all the pth entries use the same file name so that the order is more guaranteed in the virtualenv too.
build/mach_bootstrap.py
build/virtualenv_packages.txt
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -27,84 +27,16 @@ If you would like to use a different dir
 MOZBUILD_STATE_PATH environment variable to the directory you would like to
 use and re-run mach. For this change to take effect forever, you'll likely
 want to export this environment variable from your shell's init scripts.
 
 Press ENTER/RETURN to continue or CTRL+c to abort.
 '''.lstrip()
 
 
-# TODO Bug 794506 Integrate with the in-tree virtualenv configuration.
-SEARCH_PATHS = [
-    'python/mach',
-    'python/mozboot',
-    'python/mozbuild',
-    'python/mozlint',
-    'python/mozversioncontrol',
-    'python/blessings',
-    'python/compare-locales',
-    'python/configobj',
-    'python/dlmanager',
-    'python/futures',
-    'python/jsmin',
-    'python/psutil',
-    'python/pylru',
-    'python/which',
-    'python/pystache',
-    'python/pyyaml/lib',
-    'python/requests',
-    'python/slugid',
-    'python/py',
-    'python/pytest',
-    'python/pytoml',
-    'python/redo',
-    'python/voluptuous',
-    'build',
-    'build/pymake',
-    'config',
-    'dom/bindings',
-    'dom/bindings/parser',
-    'dom/media/test/external',
-    'layout/tools/reftest',
-    'other-licenses/ply',
-    'taskcluster',
-    'testing',
-    'testing/firefox-ui/harness',
-    'testing/marionette/client',
-    'testing/marionette/harness',
-    'testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py',
-    'testing/marionette/puppeteer/firefox',
-    'testing/mozbase/mozcrash',
-    'testing/mozbase/mozdebug',
-    'testing/mozbase/mozdevice',
-    'testing/mozbase/mozfile',
-    'testing/mozbase/mozhttpd',
-    'testing/mozbase/mozinfo',
-    'testing/mozbase/mozinstall',
-    'testing/mozbase/mozleak',
-    'testing/mozbase/mozlog',
-    'testing/mozbase/moznetwork',
-    'testing/mozbase/mozprocess',
-    'testing/mozbase/mozprofile',
-    'testing/mozbase/mozrunner',
-    'testing/mozbase/mozsystemmonitor',
-    'testing/mozbase/mozscreenshot',
-    'testing/mozbase/moztest',
-    'testing/mozbase/mozversion',
-    'testing/mozbase/manifestparser',
-    'testing/taskcluster',
-    'testing/tools/autotry',
-    'testing/web-platform',
-    'testing/web-platform/harness',
-    'testing/web-platform/tests/tools/wptserve',
-    'testing/web-platform/tests/tools/six',
-    'testing/xpcshell',
-    'xpcom/idl-parser',
-]
-
 # Individual files providing mach commands.
 MACH_MODULES = [
     'addon-sdk/mach_commands.py',
     'build/valgrind/mach_commands.py',
     'devtools/shared/css/generated/mach_commands.py',
     'dom/bindings/mach_commands.py',
     'dom/media/test/external/mach_commands.py',
     'layout/tools/reftest/mach_commands.py',
@@ -179,16 +111,31 @@ CATEGORIES = {
     }
 }
 
 
 # We submit data to telemetry approximately every this many mach invocations
 TELEMETRY_SUBMISSION_FREQUENCY = 10
 
 
+def search_path(mozilla_dir, packages_txt):
+    with open(os.path.join(mozilla_dir, packages_txt)) as f:
+        packages = [line.rstrip().split(':') for line in f]
+
+    for package in packages:
+        if package[0] == 'packages.txt':
+            assert len(package) == 2
+            for p in search_path(mozilla_dir, package[1]):
+                yield os.path.join(mozilla_dir, p)
+
+        if package[0].endswith('.pth'):
+            assert len(package) == 2
+            yield os.path.join(mozilla_dir, package[1])
+
+
 def bootstrap(topsrcdir, mozilla_dir=None):
     if mozilla_dir is None:
         mozilla_dir = topsrcdir
 
     # Ensure we are running Python 2.7+. We put this check here so we generate a
     # user-friendly error message rather than a cryptic stack trace on module
     # import.
     if sys.version_info[0] != 2 or sys.version_info[1] < 7:
@@ -199,17 +146,19 @@ def bootstrap(topsrcdir, mozilla_dir=Non
     # Global build system and mach state is stored in a central directory. By
     # default, this is ~/.mozbuild. However, it can be defined via an
     # environment variable. We detect first run (by lack of this directory
     # existing) and notify the user that it will be created. The logic for
     # creation is much simpler for the "advanced" environment variable use
     # case. For default behavior, we educate users and give them an opportunity
     # to react. We always exit after creating the directory because users don't
     # like surprises.
-    sys.path[0:0] = [os.path.join(mozilla_dir, path) for path in SEARCH_PATHS]
+    sys.path[0:0] = [os.path.join(mozilla_dir, path)
+                     for path in search_path(mozilla_dir,
+                                             'build/virtualenv_packages.txt')]
     import mach.main
     from mozboot.util import get_state_dir
 
     from mozbuild.util import patch_main
     patch_main()
 
     def telemetry_handler(context, data):
         # We have not opted-in to telemetry
--- a/build/virtualenv_packages.txt
+++ b/build/virtualenv_packages.txt
@@ -1,43 +1,56 @@
-marionette_driver.pth:testing/marionette/client
-marionette_harness.pth:testing/marionette/harness
-browsermobproxy.pth:testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
-six.pth:testing/web-platform/tests/tools/six
-wptserve.pth:testing/web-platform/tests/tools/wptserve
-blessings.pth:python/blessings
-configobj.pth:python/configobj
-jsmin.pth:python/jsmin
-mach.pth:python/mach
-mozbuild.pth:python/mozbuild
-mozversioncontrol.pth:python/mozversioncontrol
-mozlint.pth:python/mozlint
-pymake.pth:build/pymake
+mozilla.pth:python/mach
+mozilla.pth:python/mozboot
+mozilla.pth:python/mozbuild
+mozilla.pth:python/mozlint
+mozilla.pth:python/mozversioncontrol
+mozilla.pth:python/blessings
+mozilla.pth:python/compare-locales
+mozilla.pth:python/configobj
+mozilla.pth:python/dlmanager
+mozilla.pth:python/futures
+mozilla.pth:python/jsmin
 optional:setup.py:python/psutil:build_ext:--inplace
-optional:psutil.pth:python/psutil
-which.pth:python/which
-ply.pth:other-licenses/ply/
-mock.pth:python/mock-1.0.0
-py.pth:python/py
-pytest.pth:python/pytest
+mozilla.pth:python/psutil
+mozilla.pth:python/pylru
+mozilla.pth:python/which
+mozilla.pth:python/pystache
+mozilla.pth:python/pyyaml/lib
+mozilla.pth:python/requests
+mozilla.pth:python/slugid
+mozilla.pth:python/py
+mozilla.pth:python/pytest
+mozilla.pth:python/pytoml
+mozilla.pth:python/redo
+mozilla.pth:python/voluptuous
 mozilla.pth:build
+objdir:build
+mozilla.pth:build/pymake
 mozilla.pth:config
-mozilla.pth:xpcom/typelib/xpt/tools
 mozilla.pth:dom/bindings
 mozilla.pth:dom/bindings/parser
+mozilla.pth:dom/media/test/external
 mozilla.pth:layout/tools/reftest
-moztreedocs.pth:tools/docs
+mozilla.pth:other-licenses/ply/
+mozilla.pth:taskcluster
+mozilla.pth:testing
+mozilla.pth:testing/firefox-ui/harness
+mozilla.pth:testing/marionette/client
+mozilla.pth:testing/marionette/harness
+mozilla.pth:testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
+mozilla.pth:testing/marionette/puppeteer/firefox
 packages.txt:testing/mozbase/packages.txt
-objdir:build
-gyp.pth:media/webrtc/trunk/tools/gyp/pylib
-pyasn1.pth:python/pyasn1
-pyasn1_modules.pth:python/pyasn1-modules
-redo.pth:python/redo
-requests.pth:python/requests
-rsa.pth:python/rsa
-futures.pth:python/futures
-ecc.pth:python/PyECC
-xpcshell.pth:testing/xpcshell
-pyyaml.pth:python/pyyaml/lib
-pytoml.pth:python/pytoml
-pylru.pth:python/pylru
-taskcluster.pth:taskcluster
-dlmanager.pth:python/dlmanager
+mozilla.pth:testing/taskcluster
+mozilla.pth:testing/tools/autotry
+mozilla.pth:testing/web-platform
+mozilla.pth:testing/web-platform/harness
+mozilla.pth:testing/web-platform/tests/tools/wptserve
+mozilla.pth:testing/web-platform/tests/tools/six
+mozilla.pth:testing/xpcshell
+mozilla.pth:python/mock-1.0.0
+mozilla.pth:xpcom/typelib/xpt/tools
+mozilla.pth:tools/docs
+mozilla.pth:media/webrtc/trunk/tools/gyp/pylib
+mozilla.pth:python/pyasn1
+mozilla.pth:python/pyasn1-modules
+mozilla.pth:python/rsa
+mozilla.pth:python/PyECC