testing/mach_commands.py
author Ryan VanderMeulen <ryanvm@gmail.com>
Wed, 04 Jun 2014 16:07:06 -0400
changeset 206987 616163ca1aa528b7b418fe2edb84ac11973b851d
parent 201074 9e05b74b44a7e4bc62509298f85d4f38ff475cec
child 211698 5301f1aaf305e690126509657eff749db5d805b1
permissions -rw-r--r--
Backed out 5 changesets (bug 1015871, bug 1020257, bug 1015887, bug 1005888, bug 1015894) for desktop B2G mochitest failures on a CLOSED TREE. Backed out changeset dbcb9974b640 (bug 1020257) Backed out changeset 3220b7b2efc9 (bug 1015894) Backed out changeset 4e23647ae247 (bug 1015887) Backed out changeset 59df0cdacc15 (bug 1015871) Backed out changeset e7124b07804b (bug 1005888)

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import print_function, unicode_literals

import os

from mach.decorators import (
    CommandArgument,
    CommandProvider,
    Command,
)

from mozbuild.base import MachCommandBase


HANDLE_FILE_ERROR = '''
%s is a file.
However, I do not yet know how to run tests from files. You'll need to run
the mach command for the test type you are trying to run. e.g.
$ mach xpcshell-test path/to/file
'''.strip()

HANDLE_DIR_ERROR = '''
%s is a directory.
However, I do not yet know how to run tests from directories. You can try
running the mach command for the tests in that directory. e.g.
$ mach xpcshell-test path/to/directory
'''.strip()

UNKNOWN_TEST = '''
I don't know how to run the test: %s

You need to specify a test suite name or abbreviation. It's possible I just
haven't been told about this test suite yet. If you suspect that's the issue,
please file a bug at https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=General
and request support be added.
'''.strip()

MOCHITEST_CHUNK_BY_DIR = 4
MOCHITEST_TOTAL_CHUNKS = 5

TEST_SUITES = {
    'cppunittest': {
        'aliases': ('Cpp', 'cpp'),
        'mach_command': 'cppunittest',
        'kwargs': {'test_file': None},
    },
    'crashtest': {
        'aliases': ('C', 'Rc', 'RC', 'rc'),
        'mach_command': 'crashtest',
        'kwargs': {'test_file': None},
    },
    'crashtest-ipc': {
        'aliases': ('Cipc', 'cipc'),
        'mach_command': 'crashtest-ipc',
        'kwargs': {'test_file': None},
    },
    'jetpack': {
        'aliases': ('J',),
        'mach_command': 'jetpack-test',
        'kwargs': {},
    },
    'check-spidermonkey': {
        'aliases': ('Sm', 'sm'),
        'mach_command': 'check-spidermonkey',
        'kwargs': {'valgrind': False},
    },
    'mochitest-a11y': {
        'mach_command': 'mochitest-a11y',
        'kwargs': {'test_file': None},
    },
    'mochitest-browser': {
        'aliases': ('bc', 'BC', 'Bc'),
        'mach_command': 'mochitest-browser',
        'kwargs': {'test_file': None},
    },
    'mochitest-chrome': {
        'mach_command': 'mochitest-chrome',
        'kwargs': {'test_file': None},
    },
    'mochitest-devtools': {
        'aliases': ('dt', 'DT', 'Dt'),
        'mach_command': 'mochitest-browser --subsuite=devtools',
        'kwargs': {'test_file': None},
    },
    'mochitest-ipcplugins': {
        'make_target': 'mochitest-ipcplugins',
    },
    'mochitest-plain': {
        'mach_command': 'mochitest-plain',
        'kwargs': {'test_file': None},
    },
    'reftest': {
        'aliases': ('RR', 'rr', 'Rr'),
        'mach_command': 'reftest',
        'kwargs': {'test_file': None},
    },
    'reftest-ipc': {
        'aliases': ('Ripc',),
        'mach_command': 'reftest-ipc',
        'kwargs': {'test_file': None},
    },
    'valgrind': {
        'aliases': ('V', 'v'),
        'mach_command': 'valgrind-test',
        'kwargs': {},
    },
    'xpcshell': {
        'aliases': ('X', 'x'),
        'mach_command': 'xpcshell-test',
        'kwargs': {'test_file': 'all'},
    },
}

for i in range(1, MOCHITEST_TOTAL_CHUNKS + 1):
    TEST_SUITES['mochitest-%d' %i] = {
        'aliases': ('M%d' % i, 'm%d' % i),
        'mach_command': 'mochitest-plain',
        'kwargs': {
            'chunk_by_dir': MOCHITEST_CHUNK_BY_DIR,
            'total_chunks': MOCHITEST_TOTAL_CHUNKS,
            'this_chunk': i,
            'test_file': None,
        },
    }

TEST_HELP = '''
Test or tests to run. Tests can be specified by test suite name or alias.
The following test suites and aliases are supported: %s
''' % ', '.join(sorted(TEST_SUITES))
TEST_HELP = TEST_HELP.strip()


@CommandProvider
class Test(MachCommandBase):
    @Command('test', category='testing', description='Run tests.')
    @CommandArgument('what', default=None, nargs='*', help=TEST_HELP)
    def test(self, what):
        status = None
        for entry in what:
            status = self._run_test(entry)

            if status:
                break

        return status

    def _run_test(self, what):
        suite = None
        if what in TEST_SUITES:
            suite = TEST_SUITES[what]
        else:
            for v in TEST_SUITES.values():
                if what in v.get('aliases', []):
                    suite = v
                    break

        if suite:
            if 'mach_command' in suite:
                return self._mach_context.commands.dispatch(
                    suite['mach_command'], self._mach_context, **suite['kwargs'])

            if 'make_target' in suite:
                return self._run_make(target=suite['make_target'],
                    pass_thru=True)

            raise Exception('Do not know how to run suite. This is a logic '
                'error in this mach command.')

        if os.path.isfile(what):
            print(HANDLE_FILE_ERROR % what)
            return 1

        if os.path.isdir(what):
            print(HANDLE_DIR_ERROR % what)
            return 1

        print(UNKNOWN_TEST % what)
        return 1

@CommandProvider
class MachCommands(MachCommandBase):
    @Command('cppunittest', category='testing',
        description='Run cpp unit tests.')
    @CommandArgument('test_files', nargs='*', metavar='N',
        help='Test to run. Can be specified as one or more files or ' \
            'directories, or omitted. If omitted, the entire test suite is ' \
            'executed.')

    def run_cppunit_test(self, **params):
        import runcppunittests as cppunittests
        import logging

        if len(params['test_files']) == 0:
            testdir = os.path.join(self.distdir, 'cppunittests')
            progs = cppunittests.extract_unittests_from_args([testdir], None)
        else:
            progs = cppunittests.extract_unittests_from_args(params['test_files'], None)

        # See if we have crash symbols
        symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')
        if not os.path.isdir(symbols_path):
            symbols_path = None

        tester = cppunittests.CPPUnitTests()
        try:
            result = tester.run_tests(progs, self.bindir, symbols_path)
        except Exception, e:
            self.log(logging.ERROR, 'cppunittests',
                {'exception': str(e)},
                'Caught exception running cpp unit tests: {exception}')
            result = False

        return 0 if result else 1

@CommandProvider
class CheckSpiderMonkeyCommand(MachCommandBase):
    @Command('check-spidermonkey', category='testing', description='Run SpiderMonkey tests.')
    @CommandArgument('--valgrind', action='store_true', help='Run jit-test suite with valgrind flag')

    def run_checkspidermonkey(self, **params):
        import subprocess
        import sys

        bin_suffix = ''
        if sys.platform.startswith('win'):
            bin_suffix = '.exe'

        js = os.path.join(self.bindir, 'js%s' % bin_suffix)

        print('Running jit-tests')
        jittest_cmd = [os.path.join(self.topsrcdir, 'js', 'src', 'jit-test', 'jit_test.py'),
              js, '--no-slow', '--tbpl']
        if params['valgrind']:
            jittest_cmd.append('--valgrind')

        jittest_result = subprocess.call(jittest_cmd)

        print('running jstests')
        jstest_cmd = [os.path.join(self.topsrcdir, 'js', 'src', 'tests', 'jstests.py'),
              js, '--tbpl']
        jstest_result = subprocess.call(jstest_cmd)

        print('running jsapi-tests')
        jsapi_tests_cmd = [os.path.join(self.bindir, 'jsapi-tests%s' % bin_suffix)]
        jsapi_tests_result = subprocess.call(jsapi_tests_cmd)

        print('running check-style')
        check_style_cmd = [sys.executable, os.path.join(self.topsrcdir, 'config', 'check_spidermonkey_style.py')]
        check_style_result = subprocess.call(check_style_cmd, cwd=os.path.join(self.topsrcdir, 'js', 'src'))

        all_passed = jittest_result and jstest_result and jsapi_tests_result and check_style_result

        return all_passed