author Dave Hunt <dhunt@mozilla.com>
Tue, 14 Feb 2017 23:09:28 +0000
changeset 343302 b854735013bbbd22e800e4a6c4031475be7ff110
parent 343301 501c0a140c82d87d863eb02f5a4335e04cef7670
child 343303 9b09b5b2f7292b89dfd204f15dae6f8ebef94196
permissions -rw-r--r--
Bug 1338534 - [mozlog] Indicate unexpected pass with the correct status. r=ahal Unexpected passes can cause failures in pytest when the expected failures are set to strict. When we're logging these we want to indicate that the correct outcome. MozReview-Commit-ID: 8VoR6CTrynY

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import mozlog
import time

def pytest_addoption(parser):
    # We can't simply use mozlog.commandline.add_logging_group(parser) here because
    # Pytest's parser doesn't have the add_argument_group method Mozlog expects.
    group = parser.getgroup('mozlog')

    for name, (_class, _help) in mozlog.commandline.log_formatters.iteritems():
        group.addoption('--log-{0}'.format(name), action='append', help=_help)

    formatter_options = mozlog.commandline.fmt_options.iteritems()
    for name, (_class, _help, formatters, action) in formatter_options:
        for formatter in formatters:
            if formatter in mozlog.commandline.log_formatters:
                    '--log-{0}-{1}'.format(formatter, name),

def pytest_configure(config):
    # If using pytest-xdist for parallelization, only register plugin on master process
    if not hasattr(config, 'slaveinput'):

class MozLog(object):

    def __init__(self):
        self._started = False
        self.results = {}
        self.start_time = int(time.time() * 1000)  # in ms for Mozlog compatibility

    def _log_suite_start(self, tests):
        if not self._started:
            # As this is called for each node when using pytest-xdist, we want
            # to avoid logging multiple suite_start messages.
            self.logger.suite_start(tests=tests, time=self.start_time)
            self._started = True

    def pytest_configure(self, config):
        mozlog.commandline.setup_logging('pytest', config.known_args_namespace,
                                         defaults={}, allow_unused_options=True)
        self.logger = mozlog.get_default_logger(component='pytest')

    def pytest_sessionstart(self, session):
        '''Called before test collection; records suite start time to log later'''
        self.start_time = int(time.time() * 1000)  # in ms for Mozlog compatibility

    def pytest_collection_finish(self, session):
        '''Called after test collection is completed, just before tests are run (suite start)'''
        self._log_suite_start([item.nodeid for item in session.items])

    def pytest_xdist_node_collection_finished(self, node, ids):
        '''Called after each pytest-xdist node collection is completed'''

    def pytest_sessionfinish(self, session, exitstatus):

    def pytest_runtest_logstart(self, nodeid, location):

    def pytest_runtest_logreport(self, report):
        '''Called 3 times per test (setup, call, teardown), indicated by report.when'''
        test = report.nodeid
        status = expected = 'PASS'
        message = stack = None
        if hasattr(report, 'wasxfail'):
            expected = 'FAIL'
        if report.failed:
            status = 'FAIL' if report.when == 'call' else 'ERROR'
        if report.skipped:
            status = 'SKIP' if not hasattr(report, 'wasxfail') else 'FAIL'
        if report.longrepr is not None:
            if isinstance(report.longrepr, basestring):
                # When using pytest-xdist, longrepr is serialised as a str
                message = stack = report.longrepr
                if report.longrepr.startswith('[XPASS(strict)]'):
                    # Strict expected failures have an outcome of failed when
                    # they unexpectedly pass.
                    expected, status = ('FAIL', 'PASS')
                    # For failures, longrepr is a ReprExceptionInfo
                    crash = report.longrepr.reprcrash
                    message = "{0} (line {1})".format(crash.message, crash.lineno)
                    stack = report.longrepr.reprtraceback
                except AttributeError:
                    # For skips, longrepr is a tuple of (file, lineno, reason)
                    message = report.longrepr[-1]
        if status != expected or expected != 'PASS':
            self.results[test] = (status, expected, message, stack)
        if report.when == 'teardown':
            defaults = ('PASS', 'PASS', None, None)
            status, expected, message, stack = self.results.get(test, defaults)
            self.logger.test_end(test=test, status=status, expected=expected,
                                 message=message, stack=stack)