Bug 1368342 - Add mozlog logger that goes via a queue, r=ahal
authorJames Graham <james@hoppipolla.co.uk>
Fri, 26 May 2017 12:53:00 +0100
changeset 409412 c9d839d793c61078b9c953ad6bee96df8269bde8
parent 409411 45ed06524ff0d53dbaf58d5a5f279379d7019bb3
child 409413 0c4c63d70d4290205431d3ac7f23ce7e12749b66
push id7391
push usermtabara@mozilla.com
push dateMon, 12 Jun 2017 13:08:53 +0000
treeherdermozilla-beta@2191d7f87e2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1368342
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1368342 - Add mozlog logger that goes via a queue, r=ahal This allows subprocesses to log to a shared stream via a queue, so that we avoid the overhead of a multiprocessing Lock around all log access, but still avoid races where two processes try to log simultaneously. It's mostly useful where one process is responsible for the majority of logging, but some messages will be generated in child processes. MozReview-Commit-ID: ABl6cvpb6qI
testing/mozbase/mozlog/mozlog/proxy.py
testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
--- a/testing/mozbase/mozlog/mozlog/proxy.py
+++ b/testing/mozbase/mozlog/mozlog/proxy.py
@@ -1,13 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from .structuredlog import get_default_logger
+from multiprocessing import Queue
+from threading import Thread
+
+from .structuredlog import get_default_logger, StructuredLogger
 
 
 class ProxyLogger(object):
     """
     A ProxyLogger behaves like a
     :class:`mozlog.structuredlog.StructuredLogger`.
 
     Each method and attribute access will be forwarded to the underlying
@@ -28,8 +31,48 @@ class ProxyLogger(object):
         return getattr(self.logger, name)
 
 
 def get_proxy_logger(component=None):
     """
     Returns a :class:`ProxyLogger` for the given component.
     """
     return ProxyLogger(component)
+
+
+class QueuedProxyLogger(StructuredLogger):
+    """Logger that logs via a queue.
+
+    This is intended for multiprocessing use cases where there are
+    some subprocesses which want to share a log handler with the main thread,
+    without the overhead of having a multiprocessing lock for all logger
+    access."""
+
+    threads = {}
+
+    def __init__(self, logger):
+        StructuredLogger.__init__(self, logger.name)
+        if logger.name not in self.threads:
+            self.threads[logger.name] = LogQueueThread(Queue(), logger)
+            self.threads[logger.name].start()
+        self.queue = self.threads[logger.name].queue
+
+    def _handle_log(self, data):
+        self.queue.put(data)
+
+
+class LogQueueThread(Thread):
+    def __init__(self, queue, logger):
+        self.queue = queue
+        self.logger = logger
+        Thread.__init__(self, name="Thread-Log")
+        self.daemon = True
+
+    def run(self):
+        while True:
+            try:
+                msg = self.queue.get()
+            except (EOFError, IOError):
+                break
+            if msg is None:
+                break
+            else:
+                self.logger._handle_log(msg)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
@@ -1,17 +1,17 @@
 import json
 import os
 import multiprocessing
 import signal
 import socket
 import sys
 import time
 
-from mozlog import get_default_logger, handlers
+from mozlog import get_default_logger, handlers, proxy
 
 from wptlogging import LogLevelRewriter
 
 here = os.path.split(__file__)[0]
 
 serve = None
 sslutils = None
 
@@ -163,16 +163,18 @@ class TestEnvironment(object):
     def setup_server_logging(self):
         server_logger = get_default_logger(component="wptserve")
         assert server_logger is not None
         log_filter = handlers.LogLevelFilter(lambda x:x, "info")
         # Downgrade errors to warnings for the server
         log_filter = LogLevelRewriter(log_filter, ["error"], "warning")
         server_logger.component_filter = log_filter
 
+        server_logger = proxy.QueuedProxyLogger(server_logger)
+
         try:
             #Set as the default logger for wptserve
             serve.set_logger(server_logger)
             serve.logger = server_logger
         except Exception:
             # This happens if logging has already been set up for wptserve
             pass