Bug 1356101 - Derive the mach_bootstrap search path from build/virtualenv_packages.txt. r?gps draft
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 14 Apr 2017 08:19:08 +0900
changeset 563035 81d8b440b7f5a0ac9adb2578bc67fa42132d97f3
parent 563032 a8a33f3d81d26bfdee6fc88f15a93c32687e57ae
child 563036 1bf193c40487bc2dc9ab27ea77c31f011d8b90d1
push id54195
push userbmo:mh+mozilla@glandium.org
push dateFri, 14 Apr 2017 21:59:19 +0000
bugs1356101, 893976, 1244736
Bug 1356101 - Derive the mach_bootstrap search path from build/virtualenv_packages.txt. r?gps Most entries in virtualenv_packages.txt that are .pth or packages.txt are currently in SEARCH_PATHS in mach_bootstrap. The ones that are missing would make sense in SEARCH_PATHS. None of non-.pth or packages.txt entries, however, are in SEARCH_PATHS and don't make sense there. On the other hand, virtualenv_packages.txt misses a lot of things that are in SEARCH_PATHS, all of which should be there. One exception: xpcom/idl-parser, which causes problems due to the xpidl package containing an xpidl module, which causes problems with the in-tree scripts using it. Plus, it needs a cache directory, which is messy, so it's preferable to keep it away from the virtualenv. It turns out it was added to mach_bootstrap.py in bug 893976 for a command that was since then removed (bug 1244736), so we can get away with removing it. So instead of keeping those two separate lists out of sync, we replace the SEARCH_PATHS list from mach_bootstrap with one that is derived at runtime from the contents of virtualenv_packages.txt. And since a .pth can't fail to install in the virtualenv, it makes no sense to have psutil.pth defined as optional, which allows it to end up in the mach_bootstrap search path automatically. Finally, because we do have overlapping module names in the tree (e.g. runtests), and mach_bootstrap's SEARCH_PATHS had a guaranteed order, we change the order of the virtualenv_packages.txt file to match what used to be in mach_bootstrap, and make all the pth entries use the same file name so that the order is more guaranteed in the virtualenv too.
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -27,84 +27,16 @@ If you would like to use a different dir
 MOZBUILD_STATE_PATH environment variable to the directory you would like to
 use and re-run mach. For this change to take effect forever, you'll likely
 want to export this environment variable from your shell's init scripts.
 Press ENTER/RETURN to continue or CTRL+c to abort.
-# TODO Bug 794506 Integrate with the in-tree virtualenv configuration.
-    'python/mach',
-    'python/mozboot',
-    'python/mozbuild',
-    'python/mozlint',
-    'python/mozversioncontrol',
-    'python/blessings',
-    'python/compare-locales',
-    'python/configobj',
-    'python/dlmanager',
-    'python/futures',
-    'python/jsmin',
-    'python/psutil',
-    'python/pylru',
-    'python/which',
-    'python/pystache',
-    'python/pyyaml/lib',
-    'python/requests',
-    'python/slugid',
-    'python/py',
-    'python/pytest',
-    'python/pytoml',
-    'python/redo',
-    'python/voluptuous',
-    'build',
-    'build/pymake',
-    'config',
-    'dom/bindings',
-    'dom/bindings/parser',
-    'dom/media/test/external',
-    'layout/tools/reftest',
-    'other-licenses/ply',
-    'taskcluster',
-    'testing',
-    'testing/firefox-ui/harness',
-    'testing/marionette/client',
-    'testing/marionette/harness',
-    'testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py',
-    'testing/marionette/puppeteer/firefox',
-    'testing/mozbase/mozcrash',
-    'testing/mozbase/mozdebug',
-    'testing/mozbase/mozdevice',
-    'testing/mozbase/mozfile',
-    'testing/mozbase/mozhttpd',
-    'testing/mozbase/mozinfo',
-    'testing/mozbase/mozinstall',
-    'testing/mozbase/mozleak',
-    'testing/mozbase/mozlog',
-    'testing/mozbase/moznetwork',
-    'testing/mozbase/mozprocess',
-    'testing/mozbase/mozprofile',
-    'testing/mozbase/mozrunner',
-    'testing/mozbase/mozsystemmonitor',
-    'testing/mozbase/mozscreenshot',
-    'testing/mozbase/moztest',
-    'testing/mozbase/mozversion',
-    'testing/mozbase/manifestparser',
-    'testing/taskcluster',
-    'testing/tools/autotry',
-    'testing/web-platform',
-    'testing/web-platform/harness',
-    'testing/web-platform/tests/tools/wptserve',
-    'testing/web-platform/tests/tools/six',
-    'testing/xpcshell',
-    'xpcom/idl-parser',
 # Individual files providing mach commands.
@@ -179,16 +111,31 @@ CATEGORIES = {
 # We submit data to telemetry approximately every this many mach invocations
+def search_path(mozilla_dir, packages_txt):
+    with open(os.path.join(mozilla_dir, packages_txt)) as f:
+        packages = [line.rstrip().split(':') for line in f]
+    for package in packages:
+        if package[0] == 'packages.txt':
+            assert len(package) == 2
+            for p in search_path(mozilla_dir, package[1]):
+                yield os.path.join(mozilla_dir, p)
+        if package[0].endswith('.pth'):
+            assert len(package) == 2
+            yield os.path.join(mozilla_dir, package[1])
 def bootstrap(topsrcdir, mozilla_dir=None):
     if mozilla_dir is None:
         mozilla_dir = topsrcdir
     # Ensure we are running Python 2.7+. We put this check here so we generate a
     # user-friendly error message rather than a cryptic stack trace on module
     # import.
     if sys.version_info[0] != 2 or sys.version_info[1] < 7:
@@ -199,17 +146,19 @@ def bootstrap(topsrcdir, mozilla_dir=Non
     # Global build system and mach state is stored in a central directory. By
     # default, this is ~/.mozbuild. However, it can be defined via an
     # environment variable. We detect first run (by lack of this directory
     # existing) and notify the user that it will be created. The logic for
     # creation is much simpler for the "advanced" environment variable use
     # case. For default behavior, we educate users and give them an opportunity
     # to react. We always exit after creating the directory because users don't
     # like surprises.
-    sys.path[0:0] = [os.path.join(mozilla_dir, path) for path in SEARCH_PATHS]
+    sys.path[0:0] = [os.path.join(mozilla_dir, path)
+                     for path in search_path(mozilla_dir,
+                                             'build/virtualenv_packages.txt')]
     import mach.main
     from mozboot.util import get_state_dir
     from mozbuild.util import patch_main
     def telemetry_handler(context, data):
         # We have not opted-in to telemetry
--- a/build/virtualenv_packages.txt
+++ b/build/virtualenv_packages.txt
@@ -1,43 +1,56 @@