testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
author Andrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 15 Feb 2017 16:38:45 -0500
changeset 374494 42c74dd948b9fb34695555396daa4a6e09056198
parent 374439 29f59f3f6882445a6782a59ab55fe9473555f2ce
child 375480 47b5215fc8ad2b5f610872dffb411b82c07c0b60
permissions -rw-r--r--
Bug 1335873 - Convert marionette harness unittests to standard python unittests, r=maja_zf This formats the marionette-harness python tests to be a regular |mach python-test| suite. Though we add subsuite=marionette, this is just for automation purposes. The new preferred way to run the marionette harness tests locally is: ./mach python-test testing/marionette They will also run if running the full suite. The mozbase packages.txt file modifies mozlog to use 'setup.py' instead of 'pth'. The reason for this is that the marionette-harness tests use the pytest_mozlog pytest plugin for formatting their results (converts pytest format into something resembling the standard tbpl logging format). In order for this plugin to get picked up however, mozlog's setup.py file needs to be processed. MozReview-Commit-ID: Ata99evHxbd

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import pytest

from mock import Mock, patch, sentinel

import marionette_harness.marionette_test as marionette_test

from marionette_harness.runtests import MarionetteTestRunner, MarionetteHarness, cli


@pytest.fixture
def harness_class(request):
    """
    Mock based on MarionetteHarness whose run method just returns a number of
    failures according to the supplied test parameter
    """
    if 'num_fails_crashed' in request.funcargnames:
        num_fails_crashed = request.getfuncargvalue('num_fails_crashed')
    else:
        num_fails_crashed = (0, 0)
    harness_cls = Mock(spec=MarionetteHarness)
    harness = harness_cls.return_value
    if num_fails_crashed is None:
        harness.run.side_effect = Exception
    else:
        harness.run.return_value = sum(num_fails_crashed)
    return harness_cls


@pytest.fixture
def runner_class(request):
    """
    Mock based on MarionetteTestRunner, wherein the runner.failed,
    runner.crashed attributes are provided by a test parameter
    """
    if 'num_fails_crashed' in request.funcargnames:
        failures, crashed = request.getfuncargvalue('num_fails_crashed')
    else:
        failures = 0
        crashed = 0
    mock_runner_class = Mock(spec=MarionetteTestRunner)
    runner = mock_runner_class.return_value
    runner.failed = failures
    runner.crashed = crashed
    return mock_runner_class


@pytest.mark.parametrize(
    "num_fails_crashed,exit_code",
    [((0, 0), 0), ((1, 0), 10), ((0, 1), 10), (None, 1)],
)
def test_cli_exit_code(num_fails_crashed, exit_code, harness_class):
    with pytest.raises(SystemExit) as err:
        cli(harness_class=harness_class)
    assert err.value.code == exit_code


@pytest.mark.parametrize("num_fails_crashed", [(0, 0), (1, 0), (1, 1)])
def test_call_harness_with_parsed_args_yields_num_failures(mach_parsed_kwargs,
                                                           runner_class,
                                                           num_fails_crashed):
    with patch(
        'marionette_harness.runtests.MarionetteHarness.parse_args'
    ) as parse_args:
        failed_or_crashed = MarionetteHarness(runner_class,
                                              args=mach_parsed_kwargs).run()
        parse_args.assert_not_called()
    assert failed_or_crashed == sum(num_fails_crashed)


def test_call_harness_with_no_args_yields_num_failures(runner_class):
    with patch(
        'marionette_harness.runtests.MarionetteHarness.parse_args',
        return_value={'tests': []}
    ) as parse_args:
        failed_or_crashed = MarionetteHarness(runner_class).run()
        assert parse_args.call_count == 1
    assert failed_or_crashed == 0


def test_args_passed_to_runner_class(mach_parsed_kwargs, runner_class):
    arg_list = mach_parsed_kwargs.keys()
    arg_list.remove('tests')
    mach_parsed_kwargs.update([(a, getattr(sentinel, a)) for a in arg_list])
    harness = MarionetteHarness(runner_class, args=mach_parsed_kwargs)
    harness.process_args = Mock()
    harness.run()
    for arg in arg_list:
        assert harness._runner_class.call_args[1][arg] is getattr(sentinel, arg)


def test_harness_sets_up_default_test_handlers(mach_parsed_kwargs):
    """
    If the necessary TestCase is not in test_handlers,
    tests are omitted silently
    """
    harness = MarionetteHarness(args=mach_parsed_kwargs)
    mach_parsed_kwargs.pop('tests')
    runner = harness._runner_class(**mach_parsed_kwargs)
    assert marionette_test.MarionetteTestCase in runner.test_handlers


if __name__ == '__main__':
    import sys
    sys.exit(pytest.main(
        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))