Backed out 2 changesets (bug 1663755, bug 1663786) for causing bustages in Bug 1664106. a=backout
authorCsoregi Natalia <ncsoregi@mozilla.com>
Thu, 10 Sep 2020 15:50:24 +0300
changeset 548068 fe5de4235a06718aff9fd9fa34801916fd7fedd2
parent 548067 8948d4a41f23973ebcbeb27daebc8d281c58f31b
child 548095 e995e42082ee5432935c3db1d5e1bc502d8c20ec
push id37772
push userncsoregi@mozilla.com
push dateThu, 10 Sep 2020 12:51:47 +0000
treeherdermozilla-central@fe5de4235a06 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1663755, 1663786, 1664106
milestone82.0a1
backs out972df2adcffa91fbcc885a64c10f3ae25917b1fc
f97f4ae8e721b9070e1c4e19068084d0324b4518
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 2 changesets (bug 1663755, bug 1663786) for causing bustages in Bug 1664106. a=backout Backed out changeset 972df2adcffa (bug 1663786) Backed out changeset f97f4ae8e721 (bug 1663755)
build/build_virtualenv_packages.txt
python/mozbuild/mozbuild/virtualenv.py
--- a/build/build_virtualenv_packages.txt
+++ b/build/build_virtualenv_packages.txt
@@ -1,3 +1,4 @@
+inherit-from-parent-environment
 packages.txt:build/common_virtualenv_packages.txt
 python3:mozilla.pth:third_party/python/glean_parser
 set-variable MOZBUILD_VIRTUALENV=1
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -90,17 +90,17 @@ class VirtualenvHelper(object):
         return os.path.join(self.bin_path, binary)
 
 
 class VirtualenvManager(VirtualenvHelper):
     """Contains logic for managing virtualenvs for building the tree."""
 
     def __init__(
             self, topsrcdir, virtualenv_path, log_handle, manifest_path,
-            populate_local_paths=True):
+            parent_site_dir=None, populate_local_paths=True):
         """Create a new manager.
 
         Each manager is associated with a source directory, a path where you
         want the virtualenv to be created, and a handle to write output to.
         """
         super(VirtualenvManager, self).__init__(virtualenv_path)
 
         assert os.path.isabs(
@@ -110,16 +110,20 @@ class VirtualenvManager(VirtualenvHelper
         # Record the Python executable that was used to create the Virtualenv
         # so we can check this against sys.executable when verifying the
         # integrity of the virtualenv.
         self.exe_info_path = os.path.join(self.virtualenv_root,
                                           'python_exe.txt')
 
         self.log_handle = log_handle
         self.manifest_path = manifest_path
+        self.parent_site_dir = parent_site_dir
+        if not self.parent_site_dir:
+            import distutils.sysconfig
+            self.parent_site_dir = distutils.sysconfig.get_python_lib()
         self.populate_local_paths = populate_local_paths
 
     @property
     def virtualenv_script_path(self):
         """Path to virtualenv's own populator script."""
         return os.path.join(self.topsrcdir, 'third_party', 'python',
                             'virtualenv', 'virtualenv.py')
 
@@ -293,32 +297,42 @@ class VirtualenvManager(VirtualenvHelper
             the virtualenv site packages directory.
 
         optional -- This denotes the action as optional. The requested action
             is attempted. If it fails, we issue a warning and go on. The
             initial "optional" field is stripped then the remaining line is
             processed like normal. e.g.
             "optional:setup.py:python/foo:built_ext:-i"
 
+        copy -- Copies the given file in the virtualenv site packages
+            directory.
+
         packages.txt -- Denotes that the specified path is a child manifest. It
             will be read and processed as if its contents were concatenated
             into the manifest being read.
 
         windows -- This denotes that the action should only be taken when run
             on Windows.
 
         !windows -- This denotes that the action should only be taken when run
             on non-Windows systems.
 
         python3 -- This denotes that the action should only be taken when run
             on Python 3.
 
         python2 -- This denotes that the action should only be taken when run
             on python 2.
 
+        inherit-from-parent-environment -- This denotes that we should add the
+            configured site directory of the "parent" to the virtualenv's list
+            of site directories. This can be specified on the command line as
+            --parent-site-dir or passed in the constructor of this class. This
+            defaults to the site-packages directory of the current Python
+            interpreter if not provided.
+
         set-variable -- Set the given environment variable; e.g.
             `set-variable FOO=1`.
 
         Note that the Python interpreter running this function should be the
         one from the virtualenv. If it is the system Python or if the
         environment is not configured properly, packages could be installed
         into the wrong place. This is how virtualenv's work.
         """
@@ -327,16 +341,23 @@ class VirtualenvManager(VirtualenvHelper
         packages = self.packages()
         python_lib = distutils.sysconfig.get_python_lib()
         do_close = not bool(sitecustomize)
         sitecustomize = sitecustomize or open(
             os.path.join(os.path.dirname(os.__file__), 'sitecustomize.py'),
             mode='w')
 
         def handle_package(package):
+            if package[0] == 'inherit-from-parent-environment':
+                assert len(package) == 1
+                sitecustomize.write(
+                    'import site\n'
+                    "site.addsitedir(%s)\n" % repr(self.parent_site_dir))
+                return True
+
             if package[0].startswith('set-variable '):
                 assert len(package) == 1
                 assignment = package[0][len('set-variable '):].strip()
                 var, val = assignment.split('=', 1)
                 var = var if PY3 else ensure_binary(var)
                 val = val if PY3 else ensure_binary(val)
                 sitecustomize.write(
                     'import os\n'
@@ -346,23 +367,34 @@ class VirtualenvManager(VirtualenvHelper
             if package[0] == 'setup.py':
                 assert len(package) >= 2
 
                 self.call_setup(os.path.join(self.topsrcdir, package[1]),
                                 package[2:])
 
                 return True
 
+            if package[0] == 'copy':
+                assert len(package) == 2
+
+                src = os.path.join(self.topsrcdir, package[1])
+                dst = os.path.join(python_lib, os.path.basename(package[1]))
+
+                shutil.copy(src, dst)
+
+                return True
+
             if package[0] == 'packages.txt':
                 assert len(package) == 2
 
                 src = os.path.join(self.topsrcdir, package[1])
                 assert os.path.isfile(src), "'%s' does not exist" % src
                 submanager = VirtualenvManager(
                     self.topsrcdir, self.virtualenv_root, self.log_handle, src,
+                    parent_site_dir=self.parent_site_dir,
                     populate_local_paths=self.populate_local_paths)
                 submanager.populate(sitecustomize=sitecustomize)
 
                 return True
 
             if package[0].endswith('.pth'):
                 assert len(package) == 2
 
@@ -479,16 +511,18 @@ class VirtualenvManager(VirtualenvHelper
 
             raise Exception('Error installing package: %s' % directory)
 
     def build(self, python):
         """Build a virtualenv per tree conventions.
 
         This returns the path of the created virtualenv.
         """
+        import distutils
+
         self.create(python)
 
         # We need to populate the virtualenv using the Python executable in
         # the virtualenv for paths to be proper.
 
         # If this module was run from Python 2 then the __file__ attribute may
         # point to a Python 2 .pyc file. If we are generating a Python 3
         # virtualenv from Python 2 make sure we call Python 3 with the path to
@@ -497,17 +531,18 @@ class VirtualenvManager(VirtualenvHelper
             thismodule = __file__[:-1]
         else:
             thismodule = __file__
 
         # __PYVENV_LAUNCHER__ confuses pip about the python interpreter
         # See https://bugzilla.mozilla.org/show_bug.cgi?id=1635481
         os.environ.pop('__PYVENV_LAUNCHER__', None)
         args = [self.python_path, thismodule, 'populate', self.topsrcdir,
-                self.virtualenv_root, self.manifest_path]
+                self.virtualenv_root, self.manifest_path, '--parent-site-dir',
+                distutils.sysconfig.get_python_lib()]
         if self.populate_local_paths:
             args.append('--populate-local-paths')
 
         result = self._log_process_output(args, cwd=self.topsrcdir)
 
         if result != 0:
             raise Exception('Error populating virtualenv.')
 
@@ -682,26 +717,28 @@ if __name__ == '__main__':
     if len(sys.argv) < 2:
         print('Too few arguments', file=sys.stderr)
         sys.exit(1)
 
     parser = argparse.ArgumentParser()
     parser.add_argument('topsrcdir')
     parser.add_argument('virtualenv_path')
     parser.add_argument('manifest_path')
+    parser.add_argument('--parent-site-dir', default=None)
     parser.add_argument('--populate-local-paths', action='store_true')
 
     if sys.argv[1] == 'populate':
         # This should only be called internally.
         populate = True
         opts = parser.parse_args(sys.argv[2:])
     else:
         populate = False
         opts = parser.parse_args(sys.argv[1:])
 
     manager = VirtualenvManager(
         opts.topsrcdir, opts.virtualenv_path, sys.stdout, opts.manifest_path,
+        parent_site_dir=opts.parent_site_dir,
         populate_local_paths=opts.populate_local_paths)
 
     if populate:
         manager.populate()
     else:
         manager.ensure()