Bug 1451601 Do not buffer output in TaskCluster job output r?gps draft
authorTom Ritter <tom@mozilla.com>
Fri, 30 Mar 2018 17:18:56 -0500
changeset 777675 ad813f36f25a61dbd6c73c44eda8f3ff0518420e
parent 777674 75b397ebae934909219ad3ec44d5ebe3f0ab98ba
push id105261
push userbmo:tom@mozilla.com
push dateThu, 05 Apr 2018 04:18:27 +0000
reviewersgps
bugs1451601
milestone61.0a1
Bug 1451601 Do not buffer output in TaskCluster job output r?gps MozReview-Commit-ID: 4XOcqSXGhHi
taskcluster/taskgraph/transforms/job/mozharness.py
testing/mozharness/external_tools/tooltool.py
testing/mozharness/external_tools/virtualenv/virtualenv.py
testing/mozharness/mozharness/base/script.py
testing/mozharness/mozharness/base/signing.py
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/mozharness/scripts/android_emulator_unittest.py
old mode 100644
new mode 100755
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -148,16 +148,17 @@ def mozharness_on_docker_worker_setup(co
     env.update({
         'MOZHARNESS_CONFIG': ' '.join(run['config']),
         'MOZHARNESS_SCRIPT': run['script'],
         'MH_BRANCH': config.params['project'],
         'MH_BUILD_POOL': 'taskcluster',
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'MOZ_AUTOMATION': '1',
+        'PYTHONUNBUFFERED': '1',
     })
 
     if 'actions' in run:
         env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
 
     if 'options' in run:
         env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
 
--- a/testing/mozharness/external_tools/tooltool.py
+++ b/testing/mozharness/external_tools/tooltool.py
@@ -333,17 +333,17 @@ def digest_file(f, a):
         data = f.read(chunk_size)
     name = repr(f.name) if hasattr(f, 'name') else 'a file'
     log.debug('hashed %s with %s to be %s', name, a, h.hexdigest())
     return h.hexdigest()
 
 
 def execute(cmd):
     """Execute CMD, logging its stdout at the info level"""
-    process = Popen(cmd, shell=True, stdout=PIPE)
+    process = Popen(cmd, shell=True, stdout=PIPE, bufsize=0)
     while True:
         line = process.stdout.readline()
         if not line:
             break
         log.info(line.replace('\n', ' '))
     return process.wait() == 0
 
 
--- a/testing/mozharness/external_tools/virtualenv/virtualenv.py
+++ b/testing/mozharness/external_tools/virtualenv/virtualenv.py
@@ -660,17 +660,17 @@ def main():
         if interpreter == sys.executable:
             logger.warn('Already using interpreter %s' % interpreter)
         else:
             logger.notify('Running virtualenv with interpreter %s' % interpreter)
             env['VIRTUALENV_INTERPRETER_RUNNING'] = 'true'
             file = __file__
             if file.endswith('.pyc'):
                 file = file[:-1]
-            popen = subprocess.Popen([interpreter, file] + sys.argv[1:], env=env)
+            popen = subprocess.Popen([interpreter, file] + sys.argv[1:], env=env, bufsize=0)
             raise SystemExit(popen.wait())
 
     if not args:
         print('You must provide a DEST_DIR')
         parser.print_help()
         sys.exit(2)
     if len(args) > 1:
         print('There must be only one argument: DEST_DIR (you gave %s)' % (
@@ -743,17 +743,17 @@ def call_subprocess(cmd, show_stdout=Tru
                 env.pop(varname, None)
     else:
         env = None
     try:
         proc = subprocess.Popen(
             cmd, stderr=subprocess.STDOUT,
             stdin=None if stdin is None else subprocess.PIPE,
             stdout=stdout,
-            cwd=cwd, env=env)
+            cwd=cwd, env=env, bufsize=0)
     except Exception:
         e = sys.exc_info()[1]
         logger.fatal(
             "Error %s while executing command %s" % (e, cmd_desc))
         raise
     all_output = []
     if stdout is not None:
         if stdin is not None:
@@ -1369,17 +1369,17 @@ def install_python(home_dir, lib_dir, in
         # the value:
         py_executable = '"%s"' % py_executable
     # NOTE: keep this check as one line, cmd.exe doesn't cope with line breaks
     cmd = [py_executable, '-c', 'import sys;out=sys.stdout;'
         'getattr(out, "buffer", out).write(sys.prefix.encode("utf-8"))']
     logger.info('Testing executable with %s %s "%s"' % tuple(cmd))
     try:
         proc = subprocess.Popen(cmd,
-                            stdout=subprocess.PIPE)
+                            stdout=subprocess.PIPE, bufsize=0)
         proc_stdout, proc_stderr = proc.communicate()
     except OSError:
         e = sys.exc_info()[1]
         if e.errno == errno.EACCES:
             logger.fatal('ERROR: The executable %s could not be run: %s' % (py_executable, e))
             sys.exit(100)
         else:
             raise e
--- a/testing/mozharness/mozharness/base/script.py
+++ b/testing/mozharness/mozharness/base/script.py
@@ -1564,17 +1564,17 @@ class ScriptMixin(PlatformMixin):
             self.log("Can't open %s for writing!" % tmp_stderr_filename +
                      self.exception(), level=level)
             return None
         shell = True
         if isinstance(command, list):
             shell = False
 
         p = subprocess.Popen(command, shell=shell, stdout=tmp_stdout,
-                             cwd=cwd, stderr=tmp_stderr, env=env)
+                             cwd=cwd, stderr=tmp_stderr, env=env, bufsize=0)
         # XXX: changed from self.debug to self.log due to this error:
         #      TypeError: debug() takes exactly 1 argument (2 given)
         self.log("Temporary files: %s and %s" % (tmp_stdout_filename, tmp_stderr_filename), level=DEBUG)
         p.wait()
         tmp_stdout.close()
         tmp_stderr.close()
         return_level = DEBUG
         output = None
--- a/testing/mozharness/mozharness/base/signing.py
+++ b/testing/mozharness/mozharness/base/signing.py
@@ -114,17 +114,18 @@ class AndroidSigningMixin(object):
         # suppress_command_echo=True or something?)
         self.log("(signing %s)" % apk, level=log_level)
         try:
             p = subprocess.Popen([jarsigner, "-keystore", keystore,
                                  "-storepass", storepass,
                                  "-keypass", keypass,
                                  apk, key_alias],
                                  stdout=subprocess.PIPE,
-                                 stderr=subprocess.STDOUT)
+                                 stderr=subprocess.STDOUT,
+                                 bufsize=0)
         except OSError:
             self.exception("Error while signing %s (missing %s?):" % (apk, jarsigner))
             return -2
         except ValueError:
             self.exception("Popen called with invalid arguments during signing?")
             return -3
         parser = OutputParser(config=self.config, log_obj=self.log_obj,
                               error_list=error_list)
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -713,17 +713,17 @@ class Talos(TestingMixin, MercurialScrip
         def launch_in_debug_mode(cmdline):
             cmdline = set(cmdline)
             debug_opts = {'--debug', '--debugger', '--debugger_args'}
 
             return bool(debug_opts.intersection(cmdline))
 
         command = [python, run_tests] + options + mozlog_opts
         if launch_in_debug_mode(command):
-            talos_process = subprocess.Popen(command, cwd=self.workdir, env=env)
+            talos_process = subprocess.Popen(command, cwd=self.workdir, env=env, bufsize=0)
             talos_process.wait()
         else:
             self.return_code = self.run_command(command, cwd=self.workdir,
                                             output_timeout=output_timeout,
                                             output_parser=parser,
                                             env=env)
         if parser.minidump_output:
             self.info("Looking at the minidump files for debugging purposes...")
old mode 100644
new mode 100755
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -201,17 +201,17 @@ class AndroidEmulatorTest(TestingMixin, 
 
         command = ["emulator", "-avd", self.emulator["name"]]
         if "emulator_extra_args" in self.config:
             command += self.config["emulator_extra_args"].split()
 
         tmp_file = tempfile.NamedTemporaryFile(mode='w')
         tmp_stdout = open(tmp_file.name, 'w')
         self.info("Trying to start the emulator with this command: %s" % ' '.join(command))
-        proc = subprocess.Popen(command, stdout=tmp_stdout, stderr=tmp_stdout, env=env)
+        proc = subprocess.Popen(command, stdout=tmp_stdout, stderr=tmp_stdout, env=env, bufsize=0)
         return {
             "process": proc,
             "tmp_file": tmp_file,
         }
 
     def _retry(self, max_attempts, interval, func, description, max_time=0):
         '''
         Execute func until it returns True, up to max_attempts times, waiting for
@@ -240,17 +240,17 @@ class AndroidEmulatorTest(TestingMixin, 
         return status
 
     def _run_with_timeout(self, timeout, cmd, quiet=False):
         timeout_cmd = ['timeout', '%s' % timeout] + cmd
         return self._run_proc(timeout_cmd, quiet=quiet)
 
     def _run_proc(self, cmd, quiet=False):
         self.info('Running %s' % subprocess.list2cmdline(cmd))
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0)
         out, err = p.communicate()
         if out and not quiet:
             self.info('%s' % str(out.strip()))
         if err and not quiet:
             self.info('stderr: %s' % str(err.strip()))
         return out, err
 
     def _verify_adb(self):
@@ -339,17 +339,17 @@ class AndroidEmulatorTest(TestingMixin, 
     def _dump_emulator_log(self):
         self.info("##### %s emulator log begins" % self.emulator["name"])
         output = self.read_from_file(self.emulator_proc["tmp_file"].name, verbose=False)
         if output:
             self.info(output)
         self.info("##### %s emulator log ends" % self.emulator["name"])
 
     def _kill_processes(self, process_name):
-        p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
+        p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE, bufsize=0)
         out, err = p.communicate()
         self.info("Let's kill every process called %s" % process_name)
         for line in out.splitlines():
             if process_name in line:
                 pid = int(line.split(None, 1)[0])
                 self.info("Killing pid %d." % pid)
                 os.kill(pid, signal.SIGKILL)