mozlog: move the capture io class from web-platform/tests to mozlog (bug 1021926) r=jgraham
authorNikki S <nikkisharpley@gmail.com>
Tue, 12 Mar 2019 10:21:12 +0000
changeset 521512 f6705b8320496d0e67213299773e0c429f2b63ef
parent 521511 7f25b37567b210d3043dee744d7885aa836ecca7
child 521513 4ebf7861fca38f7b7c3b933ef0ca382888007ce8
push id10867
push userdvarga@mozilla.com
push dateThu, 14 Mar 2019 15:20:45 +0000
treeherdermozilla-beta@abad13547875 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1021926
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
mozlog: move the capture io class from web-platform/tests to mozlog (bug 1021926) r=jgraham The ability to capture the parent process' stdio is suggested to be a useful feature to move from web-platform/tests into mozlog. To do so, I have created a new capture.py file within mozlog/mozlog. This includes the CaptureIO class and its dependencies, including the LoggingWrapper and LogThread classes. These have been removed from their original location, to avoid duplication, and the files depending on them updated accordingly. It would be useful to add unittests testing the CaptureIO enter and exit methods, and the original_stdio, logging_queue and logging_thread properties. I have begun such a file with test_capture.py in mozlog/tests. This is a work in progress, however I may need some guidance, please, in regards to creating appropriate mock data to assert. Differential Revision: https://phabricator.services.mozilla.com/D22166
testing/mozbase/mozlog/mozlog/capture.py
testing/mozbase/mozlog/tests/manifest.ini
testing/mozbase/mozlog/tests/test_capture.py
testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptlogging.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozlog/mozlog/capture.py
@@ -0,0 +1,89 @@
+from __future__ import absolute_import
+
+import sys
+import threading
+from io import BytesIO
+from multiprocessing import Queue
+
+
+class LogThread(threading.Thread):
+    def __init__(self, queue, logger, level):
+        self.queue = queue
+        self.log_func = getattr(logger, level)
+        threading.Thread.__init__(self, name="Thread-Log")
+        self.daemon = True
+
+    def run(self):
+        while True:
+            try:
+                msg = self.queue.get()
+            except (EOFError, IOError):
+                break
+            if msg is None:
+                break
+            else:
+                self.log_func(msg)
+
+
+class LoggingWrapper(BytesIO):
+    """Wrapper for file like objects to redirect output to logger
+    instead"""
+
+    def __init__(self, queue, prefix=None):
+        BytesIO.__init__(self)
+        self.queue = queue
+        self.prefix = prefix
+
+    def write(self, data):
+        if isinstance(data, bytes):
+            try:
+                data = data.decode("utf8")
+            except UnicodeDecodeError:
+                data = data.decode("unicode_escape")
+
+        if data.endswith("\n"):
+            data = data[:-1]
+        if data.endswith("\r"):
+            data = data[:-1]
+        if not data:
+            return
+        if self.prefix is not None:
+            data = "%s: %s" % (self.prefix, data)
+        self.queue.put(data)
+
+    def flush(self):
+        pass
+
+
+class CaptureIO(object):
+    def __init__(self, logger, do_capture):
+        self.logger = logger
+        self.do_capture = do_capture
+        self.logging_queue = None
+        self.logging_thread = None
+        self.original_stdio = None
+
+    def __enter__(self):
+        if self.do_capture:
+            self.original_stdio = (sys.stdout, sys.stderr)
+            self.logging_queue = Queue()
+            self.logging_thread = LogThread(self.logging_queue, self.logger, "info")
+            sys.stdout = LoggingWrapper(self.logging_queue, prefix="STDOUT")
+            sys.stderr = LoggingWrapper(self.logging_queue, prefix="STDERR")
+            self.logging_thread.start()
+
+    def __exit__(self, *args, **kwargs):
+        if self.do_capture:
+            sys.stdout, sys.stderr = self.original_stdio
+            if self.logging_queue is not None:
+                self.logger.info("Closing logging queue")
+                self.logging_queue.put(None)
+                if self.logging_thread is not None:
+                    self.logging_thread.join(10)
+                while not self.logging_queue.empty():
+                    try:
+                        self.logger.warning("Dropping log message: %r", self.logging_queue.get())
+                    except Exception:
+                        pass
+                self.logging_queue.close()
+                self.logger.info("queue closed")
--- a/testing/mozbase/mozlog/tests/manifest.ini
+++ b/testing/mozbase/mozlog/tests/manifest.ini
@@ -1,6 +1,7 @@
 [DEFAULT]
 subsuite = mozbase
 [test_logger.py]
 [test_logtypes.py]
 [test_formatters.py]
 [test_structured.py]
+[test_capture.py]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozlog/tests/test_capture.py
@@ -0,0 +1,38 @@
+from __future__ import absolute_import, print_function
+import sys
+import unittest
+import mozunit
+
+from test_structured import TestHandler
+from mozlog import capture, structuredlog
+
+
+class TestCaptureIO(unittest.TestCase):
+    """Tests expected logging output of CaptureIO"""
+
+    def setUp(self):
+        self.logger = structuredlog.StructuredLogger("test")
+        self.handler = TestHandler()
+        self.logger.add_handler(self.handler)
+
+    def test_captureio_log(self):
+        """
+        CaptureIO takes in two arguments. The second argument must
+        be truthy in order for the code to run. Hence, the string
+        "capture_stdio" has been used in this test case.
+        """
+        with capture.CaptureIO(self.logger, "capture_stdio"):
+            print("message 1")
+            sys.stdout.write("message 2")
+            sys.stderr.write("message 3")
+            sys.stdout.write("\xff")
+        log = self.handler.items
+        messages = [item["message"] for item in log]
+        self.assertIn("STDOUT: message 1", messages)
+        self.assertIn("STDOUT: message 2", messages)
+        self.assertIn("STDERR: message 3", messages)
+        self.assertIn(u"STDOUT: \xff", messages)
+
+
+if __name__ == "__main__":
+    mozunit.main()
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
@@ -2,19 +2,17 @@ from __future__ import unicode_literals
 
 import multiprocessing
 import threading
 import traceback
 from Queue import Empty
 from collections import namedtuple
 from multiprocessing import Process, current_process, Queue
 
-from mozlog import structuredlog
-
-import wptlogging
+from mozlog import structuredlog, capture
 
 # Special value used as a sentinal in various commands
 Stop = object()
 
 
 class MessageLogger(object):
     def __init__(self, message_func):
         self.send_message = message_func
@@ -133,17 +131,17 @@ def start_runner(runner_command_queue, r
         runner_result_queue.put((command, args))
 
     def handle_error(e):
         logger.critical(traceback.format_exc())
         stop_flag.set()
 
     logger = MessageLogger(send_message)
 
-    with wptlogging.CaptureIO(logger, capture_stdio):
+    with capture.CaptureIO(logger, capture_stdio):
         try:
             browser = executor_browser_cls(**executor_browser_kwargs)
             executor = executor_cls(browser, **executor_kwargs)
             with TestRunner(logger, runner_command_queue, runner_result_queue, executor) as runner:
                 try:
                     runner.run()
                 except KeyboardInterrupt:
                     stop_flag.set()
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptlogging.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptlogging.py
@@ -1,13 +1,9 @@
 import logging
-import sys
-import threading
-from StringIO import StringIO
-from multiprocessing import Queue
 
 from mozlog import commandline, stdadapter, set_default_logger
 from mozlog.structuredlog import StructuredLogger
 
 def setup(args, defaults):
     logger = args.pop('log', None)
     if logger:
         set_default_logger(logger)
@@ -44,91 +40,8 @@ class LogLevelRewriter(object):
         self.from_levels = [item.upper() for item in from_levels]
         self.to_level = to_level.upper()
 
     def __call__(self, data):
         if data["action"] == "log" and data["level"].upper() in self.from_levels:
             data = data.copy()
             data["level"] = self.to_level
         return self.inner(data)
-
-
-class LogThread(threading.Thread):
-    def __init__(self, queue, logger, level):
-        self.queue = queue
-        self.log_func = getattr(logger, level)
-        threading.Thread.__init__(self, name="Thread-Log")
-        self.daemon = True
-
-    def run(self):
-        while True:
-            try:
-                msg = self.queue.get()
-            except (EOFError, IOError):
-                break
-            if msg is None:
-                break
-            else:
-                self.log_func(msg)
-
-
-class LoggingWrapper(StringIO):
-    """Wrapper for file like objects to redirect output to logger
-    instead"""
-
-    def __init__(self, queue, prefix=None):
-        StringIO.__init__(self)
-        self.queue = queue
-        self.prefix = prefix
-
-    def write(self, data):
-        if isinstance(data, str):
-            try:
-                data = data.decode("utf8")
-            except UnicodeDecodeError:
-                data = data.encode("string_escape").decode("ascii")
-
-        if data.endswith("\n"):
-            data = data[:-1]
-        if data.endswith("\r"):
-            data = data[:-1]
-        if not data:
-            return
-        if self.prefix is not None:
-            data = "%s: %s" % (self.prefix, data)
-        self.queue.put(data)
-
-    def flush(self):
-        pass
-
-
-class CaptureIO(object):
-    def __init__(self, logger, do_capture):
-        self.logger = logger
-        self.do_capture = do_capture
-        self.logging_queue = None
-        self.logging_thread = None
-        self.original_stdio = None
-
-    def __enter__(self):
-        if self.do_capture:
-            self.original_stdio = (sys.stdout, sys.stderr)
-            self.logging_queue = Queue()
-            self.logging_thread = LogThread(self.logging_queue, self.logger, "info")
-            sys.stdout = LoggingWrapper(self.logging_queue, prefix="STDOUT")
-            sys.stderr = LoggingWrapper(self.logging_queue, prefix="STDERR")
-            self.logging_thread.start()
-
-    def __exit__(self, *args, **kwargs):
-        if self.do_capture:
-            sys.stdout, sys.stderr = self.original_stdio
-            if self.logging_queue is not None:
-                self.logger.info("Closing logging queue")
-                self.logging_queue.put(None)
-                if self.logging_thread is not None:
-                    self.logging_thread.join(10)
-                while not self.logging_queue.empty():
-                    try:
-                        self.logger.warning("Dropping log message: %r", self.logging_queue.get())
-                    except Exception:
-                        pass
-                self.logging_queue.close()
-                self.logger.info("queue closed")
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
@@ -7,16 +7,17 @@ import sys
 from wptserve import sslutils
 
 import environment as env
 import products
 import testloader
 import wptcommandline
 import wptlogging
 import wpttest
+from mozlog import capture
 from font import FontInstaller
 from testrunner import ManagerGroup
 from browsers.base import NullBrowser
 
 here = os.path.split(__file__)[0]
 
 logger = None
 
@@ -127,17 +128,17 @@ def get_pause_after_test(test_loader, **
             return False
         if kwargs["repeat"] == 1 and kwargs["rerun"] == 1 and total_tests == 1:
             return True
         return False
     return kwargs["pause_after_test"]
 
 
 def run_tests(config, test_paths, product, **kwargs):
-    with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
+    with capture.CaptureIO(logger, not kwargs["no_capture_stdio"]):
         env.do_delayed_imports(logger, test_paths)
 
         product = products.load_product(config, product, load_cls=True)
 
         env_extras = product.get_env_extras(**kwargs)
 
         product.check_args(**kwargs)