Bug 1575135 - Change get_cmd_output to emit unicode strings. r=nalexander
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 20 Aug 2019 16:43:15 +0000
changeset 489120 9b73ac45f717562768abf889df36b71dc05dc735
parent 489119 f3e3d13c202ad2078d8c1e27db4970191772e74d
child 489121 2139ee18939eae14ea337e29ec46639d2b25836d
push id36465
push userdvarga@mozilla.com
push dateWed, 21 Aug 2019 16:47:43 +0000
treeherdermozilla-central@4ab60925635c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1575135 - Change get_cmd_output to emit unicode strings. r=nalexander Because most calling places in python configure don't actually want to deal with encodings, although in practical terms they should, make get_cmd_output handle it itself. Places that explicitly do want bytes can keep using subprocess directly. Differential Revision: https://phabricator.services.mozilla.com/D42604
--- a/build/moz.configure/pkg.configure
+++ b/build/moz.configure/pkg.configure
@@ -66,17 +66,17 @@ def pkg_check_modules(var, package_desc,
         log.info("checking for %s... " % package_desc)
         retcode, stdout, stderr = get_cmd_output(
             pkg_config, '--errors-to-stdout', '--print-errors', package_desc)
         if retcode == 0:
             return True
         log_writer = log.warning if allow_missing else log.error
-        with LineIO(lambda l: log_writer(l), 'replace') as o:
+        with LineIO(lambda l: log_writer(l)) as o:
         if not allow_missing:
     @depends(pkg_config, package_desc, when=package)
     @checking('%s_CFLAGS' % var, callback=lambda t: ' '.join(t))
     def pkg_cflags(pkg_config, package_desc):
         flags = check_cmd_output(pkg_config, '--cflags', package_desc)
--- a/build/moz.configure/toolchain.configure
+++ b/build/moz.configure/toolchain.configure
@@ -598,30 +598,27 @@ def check_compiler(compiler, language, t
 @imports(_from='__builtin__', _import='open')
-@imports(_from='mozbuild.util', _import='system_encoding')
 def get_vc_paths(topsrcdir):
     def vswhere(args):
         program_files = (os.environ.get('PROGRAMFILES(X86)') or
         if not program_files:
             return []
         vswhere = os.path.join(program_files, 'Microsoft Visual Studio',
                                'Installer', 'vswhere.exe')
         if not os.path.exists(vswhere):
             return []
-        return json.loads(
-            check_cmd_output(vswhere, '-format', 'json', *args)
-            .decode(system_encoding, 'replace'))
+        return json.loads(check_cmd_output(vswhere, '-format', 'json', *args))
     for install in vswhere(['-products', '*', '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64']):
         path = install['installationPath']
         tools_version = open(os.path.join(
             path, r'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt'), 'rb').read().strip()
         tools_path = os.path.join(
             path, r'VC\Tools\MSVC', tools_version)
         yield (Version(install['installationVersion']), tools_path)
--- a/build/moz.configure/util.configure
+++ b/build/moz.configure/util.configure
@@ -21,16 +21,17 @@ def configure_error(message):
 # A wrapper to obtain a process' output and return code.
 # Returns a tuple (retcode, stdout, stderr).
 @imports(_from='__builtin__', _import='unicode')
 @imports(_from='mozbuild.shellutil', _import='quote')
+@imports(_from='mozbuild.util', _import='system_encoding')
 def get_cmd_output(*args, **kwargs):
     # subprocess on older Pythons can't handle unicode keys or values in
     # environment dicts. Normalize automagically so callers don't have to
     # deal with this.
     if 'env' in kwargs:
         normalized_env = {}
         for k, v in kwargs['env'].items():
             if isinstance(k, unicode):
@@ -48,16 +49,18 @@ def get_cmd_output(*args, **kwargs):
                             # On Python 2 on Windows, close_fds prevents the
                             # process from inheriting stdout/stderr.
                             # Elsewhere, it simply prevents it from inheriting
                             # extra file descriptors, which is what we want.
                             close_fds=os.name != 'nt',
     stdout, stderr = proc.communicate()
+    stdout = stdout.decode(system_encoding, 'replace')
+    stderr = stderr.decode(system_encoding, 'replace')
     return proc.wait(), stdout, stderr
 # A wrapper to obtain a process' output that returns the output generated
 # by running the given command if it exits normally, and streams that
 # output to log.debug and calls die or the given error callback if it
 # does not.
 @imports(_from='mozbuild.configure.util', _import='LineIO')
@@ -70,17 +73,17 @@ def check_cmd_output(*args, **kwargs):
         if retcode == 0:
             return stdout
         log.debug('The command returned non-zero exit status %d.',
         for out, desc in ((stdout, 'output'), (stderr, 'error output')):
             if out:
                 log.debug('Its %s was:', desc)
-                with LineIO(lambda l: log.debug('| %s', l), 'replace') as o:
+                with LineIO(lambda l: log.debug('| %s', l)) as o:
         if onerror:
             return onerror()
         die('Command `%s` failed with exit status %d.' %
             (quote(*args), retcode))