tools/tryselect/test/test_mozharness_integration.py
author Andrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 22 Apr 2019 22:32:34 +0000
changeset 470488 60c512bab3e986b609d7fda44d827c21d90612d2
parent 469291 483960ad2be29aa30a348c57cd817532edc18d40
child 471317 d1240d47e791648b1fafa798373ae12df02ea285
permissions -rw-r--r--
Bug 1523303 - Align mozharness suite names with the ones in 'moztest.resolve', r=gbrown This officially makes 'moztest.resolve' the source of truth when it comes to suite names. It aligns that file with the names used in both the desktop_unittest and android_emulator_unittest scripts. Differential Revision: https://phabricator.services.mozilla.com/D27555

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

import imp
import json
import os

import mozunit
import pytest

from tryselect.tasks import build, resolve_tests_by_suite

MOZHARNESS_SCRIPTS = {
    'android_emulator_unittest': {
        'class_name': 'AndroidEmulatorTest',
        'configs': [
            'android/android_common.py',
        ],
        'xfail': [
            'cppunittest',
            'gtest',
            'geckoview-junit',
            'jittest',
            'jsreftest',
        ],
    },
    'desktop_unittest': {
        'class_name': 'DesktopUnittest',
        'configs': [
            'unittests/linux_unittest.py',
            'unittests/mac_unittest.py',
            'unittests/win_unittest.py',
        ],
        'xfail': [
            'cppunittest',
            'gtest',
            'jittest',
            'jittest-chunked',
            'jittest1',
            'jittest2',
            'jsreftest',
            'mochitest-browser-chrome-instrumentation',
            'mochitest-devtools-chrome-webreplay',
            'mochitest-valgrind-plain',
            'reftest-gpu',
            'reftest-no-accel',
            'reftest-qr',
        ],
    },
}
"""A suite being listed in a script's `xfail` list  means it won't work
properly with MOZHARNESS_TEST_PATHS (the mechanism |mach try fuzzy <path>|
uses).
"""


def get_mozharness_test_paths(name):
    scriptdir = os.path.join(build.topsrcdir, 'testing', 'mozharness', 'scripts')

    files = imp.find_module(name, [scriptdir])
    mod = imp.load_module('scripts.{}'.format(name), *files)

    class_name = MOZHARNESS_SCRIPTS[name]['class_name']
    cls = getattr(mod, class_name)
    return cls(require_config_file=False)._get_mozharness_test_paths


@pytest.fixture(scope='module')
def all_suites():
    from moztest.resolve import _test_flavors, _test_subsuites
    all_suites = []
    for flavor in _test_flavors:
        all_suites.append({'flavor': flavor, 'srcdir_relpath': 'test'})

    for flavor, subsuite in _test_subsuites:
        all_suites.append({'flavor': flavor, 'subsuite': subsuite, 'srcdir_relpath': 'test'})

    return all_suites


def generate_suites_from_config(path):
    configdir = os.path.join(build.topsrcdir, 'testing', 'mozharness', 'configs')

    parent, name = os.path.split(path)
    name = os.path.splitext(name)[0]

    files = imp.find_module('{}'.format(name), [os.path.join(configdir, parent)])
    mod = imp.load_module('config.{}'.format(name), *files)
    config = mod.config

    for category in sorted(config['suite_definitions']):
        if category == 'mozmill':
            continue

        key = 'all_{}_suites'.format(category)
        if key not in config:
            yield category,
            continue

        for suite in sorted(config['all_{}_suites'.format(category)]):
            yield category, suite


def generate_suites():
    for name, script in MOZHARNESS_SCRIPTS.items():
        seen = set()

        for path in script['configs']:
            for suite in generate_suites_from_config(path):
                if suite in seen:
                    continue
                seen.add(suite)

                item = (name, suite)

                if suite[-1] in script['xfail']:
                    item = pytest.param(item, marks=pytest.mark.xfail)

                yield item


def idfn(item):
    name, suite = item
    return "{}/{}".format(name, suite[-1])


@pytest.mark.parametrize('item', generate_suites(), ids=idfn)
def test_suites(item, patch_resolver, all_suites):
    """An integration test to make sure the suites returned by
    `tasks.resolve_tests_by_suite` match up with the names defined in
    mozharness.
    """
    patch_resolver([], all_suites)
    suites = resolve_tests_by_suite(['test'])
    os.environ['MOZHARNESS_TEST_PATHS'] = json.dumps(suites)

    name, suite = item
    func = get_mozharness_test_paths(name)
    assert func(*suite)


if __name__ == '__main__':
    mozunit.main()