Bug 1520394 - Wrap the os and subprocess modules in python configure sandbox. r=nalexander
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 16 Jan 2019 23:39:10 +0000
changeset 514179 adc8ba2d499e
parent 514178 e1f95cea7f0c
child 514180 ab27c46876dd
push id1953
push userffxbld-merge
push dateMon, 11 Mar 2019 12:10:20 +0000
treeherdermozilla-release@9c35dcbaa899 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1520394
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1520394 - Wrap the os and subprocess modules in python configure sandbox. r=nalexander Because the sandbox has its own environment from which it initializes, os.environ should reflect that in the sandbox. And the few obvious things that use os.environ too, i.e. subprocess.*. Depends on D16667 Differential Revision: https://phabricator.services.mozilla.com/D16668
python/mozbuild/mozbuild/configure/__init__.py
python/mozbuild/mozbuild/test/configure/common.py
--- a/python/mozbuild/mozbuild/configure/__init__.py
+++ b/python/mozbuild/mozbuild/configure/__init__.py
@@ -25,18 +25,20 @@ from mozbuild.configure.options import (
 )
 from mozbuild.configure.help import HelpFormatter
 from mozbuild.configure.util import (
     ConfigureOutputHandler,
     getpreferredencoding,
     LineIO,
 )
 from mozbuild.util import (
+    encode,
     exec_,
     memoize,
+    memoized_property,
     ReadOnlyDict,
     ReadOnlyNamespace,
 )
 
 import mozpack.path as mozpath
 
 
 class ConfigureError(Exception):
@@ -289,16 +291,18 @@ class ConfigureSandbox(dict):
         for k in ('abspath', 'basename', 'dirname', 'isabs', 'join',
                   'normcase', 'normpath', 'realpath', 'relpath')
     }))
 
     def __init__(self, config, environ=os.environ, argv=sys.argv,
                  stdout=sys.stdout, stderr=sys.stderr, logger=None):
         dict.__setitem__(self, '__builtins__', self.BUILTINS)
 
+        self._environ = dict(environ)
+
         self._paths = []
         self._all_paths = set()
         self._templates = set()
         # Associate SandboxDependsFunctions to DependsFunctions.
         self._depends = OrderedDict()
         self._seen = set()
         # Store the @imports added to a given function.
         self._imports = {}
@@ -819,26 +823,62 @@ class ConfigureSandbox(dict):
         for _from, _import, _as in self._imports.pop(func, ()):
             _from = '%s.' % _from if _from else ''
             if _as:
                 glob[_as] = self._get_one_import('%s%s' % (_from, _import))
             else:
                 what = _import.split('.')[0]
                 glob[what] = self._get_one_import('%s%s' % (_from, what))
 
+    @memoized_property
+    def _wrapped_os(self):
+        wrapped_os = {}
+        exec_('from os import *', {}, wrapped_os)
+        wrapped_os['environ'] = self._environ
+        return ReadOnlyNamespace(**wrapped_os)
+
+    @memoized_property
+    def _wrapped_subprocess(self):
+        wrapped_subprocess = {}
+        exec_('from subprocess import *', {}, wrapped_subprocess)
+
+        def wrap(function):
+            def wrapper(*args, **kwargs):
+                if 'env' not in kwargs:
+                    kwargs['env'] = encode(self._environ)
+                return function(*args, **kwargs)
+            return wrapper
+
+        for f in ('call', 'check_call', 'check_output', 'Popen'):
+            wrapped_subprocess[f] = wrap(wrapped_subprocess[f])
+
+        return ReadOnlyNamespace(**wrapped_subprocess)
+
     def _get_one_import(self, what):
         # The special `__sandbox__` module gives access to the sandbox
         # instance.
         if what == '__sandbox__':
             return self
         # Special case for the open() builtin, because otherwise, using it
         # fails with "IOError: file() constructor not accessible in
         # restricted mode"
         if what == '__builtin__.open':
             return lambda *args, **kwargs: open(*args, **kwargs)
+        # Special case os and os.environ so that os.environ is our copy of
+        # the environment.
+        if what == 'os.environ':
+            return self._environ
+        if what == 'os':
+            return self._wrapped_os
+        # And subprocess, so that its functions use our os.environ
+        if what == 'subprocess':
+            return self._wrapped_subprocess
+        if what in ('subprocess.call', 'subprocess.check_call',
+                    'subprocess.check_output', 'subprocess.Popen'):
+            return getattr(self._wrapped_subprocess, what[len('subprocess.'):])
         # Until this proves to be a performance problem, just construct an
         # import statement and execute it.
         import_line = ''
         if '.' in what:
             _from, what = what.rsplit('.', 1)
             import_line += 'from %s ' % _from
         import_line += 'import %s as imported' % what
         glob = {}
--- a/python/mozbuild/mozbuild/test/configure/common.py
+++ b/python/mozbuild/mozbuild/test/configure/common.py
@@ -77,22 +77,21 @@ class ConfigureTestSandbox(ConfigureSand
         self._search_path = environ.get('PATH', '').split(os.pathsep)
 
         self._subprocess_paths = {
             mozpath.abspath(k): v for k, v in paths.iteritems() if v
         }
 
         paths = paths.keys()
 
-        environ = dict(environ)
+        environ = copy.copy(environ)
         if 'CONFIG_SHELL' not in environ:
             environ['CONFIG_SHELL'] = mozpath.abspath('/bin/sh')
             self._subprocess_paths[environ['CONFIG_SHELL']] = self.shell
             paths.append(environ['CONFIG_SHELL'])
-        self._environ = copy.copy(environ)
         self._subprocess_paths[mozpath.join(topsrcdir, 'build/win32/vswhere.exe')] = self.vswhere
 
         vfs = ConfigureTestVFS(paths)
 
         os_path = {
             k: getattr(vfs, k) for k in dir(vfs) if not k.startswith('_')
         }
 
@@ -129,19 +128,16 @@ class ConfigureTestSandbox(ConfigureSand
             return self.imported_os.path
 
         if what == 'os.path.exists':
             return self.imported_os.path.exists
 
         if what == 'os.path.isfile':
             return self.imported_os.path.isfile
 
-        if what == 'os.environ':
-            return self._environ
-
         if what == 'ctypes.wintypes':
             return ReadOnlyNamespace(
                 LPCWSTR=0,
                 LPWSTR=1,
                 DWORD=2,
             )
 
         if what == 'ctypes':