Bug 1428713 - [mozprocess] Add support for Python 3 r=davehunt
authorPavel Slepushkin <slepushkin@yandex.ru>
Sat, 04 Aug 2018 17:32:57 +0200
changeset 431778 aaabe6d53de26960892d20922454b64708ad2319
parent 431777 6560b92dbe64151387d6dd2e73167912a17d3af2
child 431779 20b991052ad7bf4dcd30b34b7f49397bb789e2da
push id106548
push userdvarga@mozilla.com
push dateWed, 15 Aug 2018 22:25:34 +0000
treeherdermozilla-inbound@baba90c7c28f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1428713
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1428713 - [mozprocess] Add support for Python 3 r=davehunt MozReview-Commit-ID: 9wHoIEboA0K
testing/mozbase/mozprocess/mozprocess/processhandler.py
testing/mozbase/mozprocess/setup.py
testing/mozbase/mozprocess/tests/manifest.ini
testing/mozbase/mozprocess/tests/proclaunch.py
testing/mozbase/mozprocess/tests/test_output.py
testing/mozbase/mozprocess/tests/test_process_reader.py
testing/mozbase/mozrunner/setup.py
testing/tps/setup.py
--- a/testing/mozbase/mozprocess/mozprocess/processhandler.py
+++ b/testing/mozbase/mozprocess/mozprocess/processhandler.py
@@ -8,17 +8,20 @@ import errno
 import os
 import signal
 import subprocess
 import sys
 import threading
 import time
 import traceback
 
-from Queue import Queue, Empty
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty
 from datetime import datetime
 
 
 __all__ = ['ProcessHandlerMixin', 'ProcessHandler', 'LogOutput',
            'StoreOutput', 'StreamOutput']
 
 # Set the MOZPROCESS_DEBUG environment variable to 1 to see some debugging output
 MOZPROCESS_DEBUG = os.getenv("MOZPROCESS_DEBUG")
@@ -119,24 +122,24 @@ class ProcessHandlerMixin(object):
                 raise
 
         def debug(self, msg):
             if not MOZPROCESS_DEBUG:
                 return
             thread = threading.current_thread().name
             print("DBG::MOZPROC PID:{} ({}) | {}".format(self.pid, thread, msg))
 
-        def __del__(self, _maxint=sys.maxint):
+        def __del__(self, _maxint=sys.maxsize):
             if isWin:
                 handle = getattr(self, '_handle', None)
                 if handle:
                     if hasattr(self, '_internal_poll'):
                         self._internal_poll(_deadstate=_maxint)
                     else:
-                        self.poll(_deadstate=sys.maxint)
+                        self.poll(_deadstate=sys.maxsize)
                 if handle or self._job or self._io_port:
                     self._cleanup()
             else:
                 subprocess.Popen.__del__(self)
 
         def kill(self, sig=None):
             if isWin:
                 try:
@@ -1064,17 +1067,17 @@ class StoreOutput(object):
 class StreamOutput(object):
     """pass output to a stream and flush"""
 
     def __init__(self, stream):
         self.stream = stream
 
     def __call__(self, line):
         try:
-            self.stream.write(line + '\n')
+            self.stream.write(line.decode() + '\n')
         except UnicodeDecodeError:
             # TODO: Workaround for bug #991866 to make sure we can display when
             # when normal UTF-8 display is failing
             self.stream.write(line.decode('iso8859-1') + '\n')
         self.stream.flush()
 
 
 class LogOutput(StreamOutput):
--- a/testing/mozbase/mozprocess/setup.py
+++ b/testing/mozbase/mozprocess/setup.py
@@ -1,28 +1,29 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 from setuptools import setup
 
-PACKAGE_VERSION = '0.26'
+PACKAGE_VERSION = '1.0.0'
 
 setup(name='mozprocess',
       version=PACKAGE_VERSION,
       description="Mozilla-authored process handling",
       long_description="see https://firefox-source-docs.mozilla.org/mozbase/index.html",
       classifiers=['Environment :: Console',
                    'Intended Audience :: Developers',
                    'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
                    'Natural Language :: English',
                    'Operating System :: OS Independent',
-                   'Programming Language :: Python',
+                   'Programming Language :: Python :: 2.7',
+                   'Programming Language :: Python :: 3.5'
                    'Topic :: Software Development :: Libraries :: Python Modules',
                    ],
       keywords='mozilla',
       author='Mozilla Automation and Tools team',
       author_email='tools@lists.mozilla.org',
       url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase',
       license='MPL 2.0',
       packages=['mozprocess'],
--- a/testing/mozbase/mozprocess/tests/manifest.ini
+++ b/testing/mozbase/mozprocess/tests/manifest.ini
@@ -1,10 +1,9 @@
 [DEFAULT]
 subsuite = mozbase, os == "linux"
-skip-if = python == 3
 [test_kill.py]
 [test_misc.py]
 [test_poll.py]
 [test_wait.py]
 [test_output.py]
 [test_params.py]
 [test_process_reader.py]
--- a/testing/mozbase/mozprocess/tests/proclaunch.py
+++ b/testing/mozbase/mozprocess/tests/proclaunch.py
@@ -1,15 +1,18 @@
 #!/usr/bin/env python
 
 from __future__ import absolute_import, print_function
 
 import argparse
 import collections
-import ConfigParser
+try:
+    import configparser as ConfigParser
+except ImportError:
+    import ConfigParser
 import multiprocessing
 import time
 
 ProcessNode = collections.namedtuple('ProcessNode', ['maxtime', 'children'])
 
 
 class ProcessLauncher(object):
 
--- a/testing/mozbase/mozprocess/tests/test_output.py
+++ b/testing/mozbase/mozprocess/tests/test_output.py
@@ -1,19 +1,19 @@
 #!/usr/bin/env python
 
 from __future__ import absolute_import
 
-import io
 import os
 
 import mozunit
 
 import proctest
 from mozprocess import processhandler
+import six
 
 here = os.path.dirname(os.path.abspath(__file__))
 
 
 class ProcTestOutput(proctest.ProcTest):
     """ Class to test operations related to output handling """
 
     def test_process_output_twice(self):
@@ -44,34 +44,32 @@ class ProcTestOutput(proctest.ProcTest):
         self.determine_status(p, False, ())
 
     def test_stream_process_output(self):
         """
         Process output stream does not buffer
         """
         expected = '\n'.join([str(n) for n in range(0, 10)])
 
-        stream = io.BytesIO()
-        buf = io.BufferedRandom(stream)
+        stream = six.StringIO()
 
         p = processhandler.ProcessHandler([self.python,
                                            os.path.join("scripts", "proccountfive.py")],
                                           cwd=here,
-                                          stream=buf)
+                                          stream=stream)
 
         p.run()
         p.wait()
         for i in range(5, 10):
             stream.write(str(i) + '\n')
 
-        buf.flush()
         self.assertEquals(stream.getvalue().strip(), expected)
 
         # make sure mozprocess doesn't close the stream
         # since mozprocess didn't create it
-        self.assertFalse(buf.closed)
-        buf.close()
+        self.assertFalse(stream.closed)
+        stream.close()
 
         self.determine_status(p, False, ())
 
 
 if __name__ == '__main__':
     mozunit.main()
--- a/testing/mozbase/mozprocess/tests/test_process_reader.py
+++ b/testing/mozbase/mozprocess/tests/test_process_reader.py
@@ -28,38 +28,38 @@ class TestProcessReader(unittest.TestCas
         def on_timeout():
             self.timeout = True
         self.reader = ProcessReader(stdout_callback=self.out,
                                     stderr_callback=self.err,
                                     finished_callback=on_finished,
                                     timeout_callback=on_timeout)
 
     def test_stdout_callback(self):
-        proc = run_python('print 1; print 2')
+        proc = run_python('print(1); print(2)')
         self.reader.start(proc)
         self.reader.thread.join()
 
-        self.assertEqual(self.out.output, ['1', '2'])
+        self.assertEqual(self.out.output, [b'1', b'2'])
         self.assertEqual(self.err.output, [])
 
     def test_stderr_callback(self):
         proc = run_python('import sys; sys.stderr.write("hello world\\n")')
         self.reader.start(proc)
         self.reader.thread.join()
 
         self.assertEqual(self.out.output, [])
-        self.assertEqual(self.err.output, ['hello world'])
+        self.assertEqual(self.err.output, [b'hello world'])
 
     def test_stdout_and_stderr_callbacks(self):
-        proc = run_python('import sys; sys.stderr.write("hello world\\n"); print 1; print 2')
+        proc = run_python('import sys; sys.stderr.write("hello world\\n"); print(1); print(2)')
         self.reader.start(proc)
         self.reader.thread.join()
 
-        self.assertEqual(self.out.output, ['1', '2'])
-        self.assertEqual(self.err.output, ['hello world'])
+        self.assertEqual(self.out.output, [b'1', b'2'])
+        self.assertEqual(self.err.output, [b'hello world'])
 
     def test_finished_callback(self):
         self.assertFalse(self.finished)
         proc = run_python('')
         self.reader.start(proc)
         self.reader.thread.join()
         self.assertTrue(self.finished)
 
@@ -80,28 +80,28 @@ class TestProcessReader(unittest.TestCas
         self.reader.thread.join()
         self.assertTrue(self.timeout)
         self.assertFalse(self.finished)
 
     def test_read_without_eol(self):
         proc = run_python('import sys; sys.stdout.write("1")')
         self.reader.start(proc)
         self.reader.thread.join()
-        self.assertEqual(self.out.output, ['1'])
+        self.assertEqual(self.out.output, [b'1'])
 
     def test_read_with_strange_eol(self):
         proc = run_python('import sys; sys.stdout.write("1\\r\\r\\r\\n")')
         self.reader.start(proc)
         self.reader.thread.join()
-        self.assertEqual(self.out.output, ['1'])
+        self.assertEqual(self.out.output, [b'1'])
 
     def test_mixed_stdout_stderr(self):
-        proc = run_python('import sys; sys.stderr.write("hello world\\n"); print 1; print 2',
+        proc = run_python('import sys; sys.stderr.write("hello world\\n"); print(1); print(2)',
                           stderr=subprocess.STDOUT)
         self.reader.start(proc)
         self.reader.thread.join()
 
-        self.assertEqual(sorted(self.out.output), sorted(['1', '2', 'hello world']))
+        self.assertEqual(sorted(self.out.output), sorted([b'1', b'2', b'hello world']))
         self.assertEqual(self.err.output, [])
 
 
 if __name__ == '__main__':
     mozunit.main()
--- a/testing/mozbase/mozrunner/setup.py
+++ b/testing/mozbase/mozrunner/setup.py
@@ -11,17 +11,17 @@ PACKAGE_VERSION = '7.0.1'
 
 desc = """Reliable start/stop/configuration of Mozilla Applications (Firefox, Thunderbird, etc.)"""
 
 deps = [
     'mozdevice>=1.*',
     'mozfile==1.*',
     'mozinfo>=0.7,<2',
     'mozlog==3.*',
-    'mozprocess>=0.23,<1',
+    'mozprocess>=0.23,<2',
     'mozprofile>=1.1.0,<2',
     'six>=1.10.0,<2',
 ]
 
 EXTRAS_REQUIRE = {'crash': ['mozcrash >= 1.0']}
 
 
 setup(name=PACKAGE_NAME,
--- a/testing/tps/setup.py
+++ b/testing/tps/setup.py
@@ -8,17 +8,17 @@ import sys
 
 version = '0.6'
 
 deps = ['httplib2 == 0.9.2',
         'mozfile == 1.2',
         'mozhttpd == 0.7',
         'mozinfo >= 0.10',
         'mozinstall == 1.16',
-        'mozprocess == 0.26',
+        'mozprocess == 1.0.0',
         'mozprofile == 1.1.0',
         'mozrunner == 7.0.1',
         'mozversion == 1.5',
        ]
 
 # we only support python 2.6+ right now
 assert sys.version_info[0] == 2
 assert sys.version_info[1] >= 6