testing/mach_commands.py
author Wan-Teh Chang <wtc@google.com>
Thu, 05 Jun 2014 07:06:32 -0700
changeset 206065 189492a9a1151073e187dd62440011c8fee2b5fe
parent 199990 9e05b74b44a7e4bc62509298f85d4f38ff475cec
child 210576 5301f1aaf305e690126509657eff749db5d805b1
permissions -rw-r--r--
Bug 1020695: Update Mozilla to use NSS 3.16.2 Beta 3. Includes fixes for bug 1013088, bug 996237, bug 970539, bug 1016567, bug 485732, bug 334013, bug 959864, bug 1016836, bug 1016811, bug 1018536, bug 996250, bug 1009227, bug 963150.

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import print_function, unicode_literals

import os

from mach.decorators import (
    CommandArgument,
    CommandProvider,
    Command,
)

from mozbuild.base import MachCommandBase


HANDLE_FILE_ERROR = '''
%s is a file.
However, I do not yet know how to run tests from files. You'll need to run
the mach command for the test type you are trying to run. e.g.
$ mach xpcshell-test path/to/file
'''.strip()

HANDLE_DIR_ERROR = '''
%s is a directory.
However, I do not yet know how to run tests from directories. You can try
running the mach command for the tests in that directory. e.g.
$ mach xpcshell-test path/to/directory
'''.strip()

UNKNOWN_TEST = '''
I don't know how to run the test: %s

You need to specify a test suite name or abbreviation. It's possible I just
haven't been told about this test suite yet. If you suspect that's the issue,
please file a bug at https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=General
and request support be added.
'''.strip()

MOCHITEST_CHUNK_BY_DIR = 4
MOCHITEST_TOTAL_CHUNKS = 5

TEST_SUITES = {
    'cppunittest': {
        'aliases': ('Cpp', 'cpp'),
        'mach_command': 'cppunittest',
        'kwargs': {'test_file': None},
    },
    'crashtest': {
        'aliases': ('C', 'Rc', 'RC', 'rc'),
        'mach_command': 'crashtest',
        'kwargs': {'test_file': None},
    },
    'crashtest-ipc': {
        'aliases': ('Cipc', 'cipc'),
        'mach_command': 'crashtest-ipc',
        'kwargs': {'test_file': None},
    },
    'jetpack': {
        'aliases': ('J',),
        'mach_command': 'jetpack-test',
        'kwargs': {},
    },
    'check-spidermonkey': {
        'aliases': ('Sm', 'sm'),
        'mach_command': 'check-spidermonkey',
        'kwargs': {'valgrind': False},
    },
    'mochitest-a11y': {
        'mach_command': 'mochitest-a11y',
        'kwargs': {'test_file': None},
    },
    'mochitest-browser': {
        'aliases': ('bc', 'BC', 'Bc'),
        'mach_command': 'mochitest-browser',
        'kwargs': {'test_file': None},
    },
    'mochitest-chrome': {
        'mach_command': 'mochitest-chrome',
        'kwargs': {'test_file': None},
    },
    'mochitest-devtools': {
        'aliases': ('dt', 'DT', 'Dt'),
        'mach_command': 'mochitest-browser --subsuite=devtools',
        'kwargs': {'test_file': None},
    },
    'mochitest-ipcplugins': {
        'make_target': 'mochitest-ipcplugins',
    },
    'mochitest-plain': {
        'mach_command': 'mochitest-plain',
        'kwargs': {'test_file': None},
    },
    'reftest': {
        'aliases': ('RR', 'rr', 'Rr'),
        'mach_command': 'reftest',
        'kwargs': {'test_file': None},
    },
    'reftest-ipc': {
        'aliases': ('Ripc',),
        'mach_command': 'reftest-ipc',
        'kwargs': {'test_file': None},
    },
    'valgrind': {
        'aliases': ('V', 'v'),
        'mach_command': 'valgrind-test',
        'kwargs': {},
    },
    'xpcshell': {
        'aliases': ('X', 'x'),
        'mach_command': 'xpcshell-test',
        'kwargs': {'test_file': 'all'},
    },
}

for i in range(1, MOCHITEST_TOTAL_CHUNKS + 1):
    TEST_SUITES['mochitest-%d' %i] = {
        'aliases': ('M%d' % i, 'm%d' % i),
        'mach_command': 'mochitest-plain',
        'kwargs': {
            'chunk_by_dir': MOCHITEST_CHUNK_BY_DIR,
            'total_chunks': MOCHITEST_TOTAL_CHUNKS,
            'this_chunk': i,
            'test_file': None,
        },
    }

TEST_HELP = '''
Test or tests to run. Tests can be specified by test suite name or alias.
The following test suites and aliases are supported: %s
''' % ', '.join(sorted(TEST_SUITES))
TEST_HELP = TEST_HELP.strip()


@CommandProvider
class Test(MachCommandBase):
    @Command('test', category='testing', description='Run tests.')
    @CommandArgument('what', default=None, nargs='*', help=TEST_HELP)
    def test(self, what):
        status = None
        for entry in what:
            status = self._run_test(entry)

            if status:
                break

        return status

    def _run_test(self, what):
        suite = None
        if what in TEST_SUITES:
            suite = TEST_SUITES[what]
        else:
            for v in TEST_SUITES.values():
                if what in v.get('aliases', []):
                    suite = v
                    break

        if suite:
            if 'mach_command' in suite:
                return self._mach_context.commands.dispatch(
                    suite['mach_command'], self._mach_context, **suite['kwargs'])

            if 'make_target' in suite:
                return self._run_make(target=suite['make_target'],
                    pass_thru=True)

            raise Exception('Do not know how to run suite. This is a logic '
                'error in this mach command.')

        if os.path.isfile(what):
            print(HANDLE_FILE_ERROR % what)
            return 1

        if os.path.isdir(what):
            print(HANDLE_DIR_ERROR % what)
            return 1

        print(UNKNOWN_TEST % what)
        return 1

@CommandProvider
class MachCommands(MachCommandBase):
    @Command('cppunittest', category='testing',
        description='Run cpp unit tests.')
    @CommandArgument('test_files', nargs='*', metavar='N',
        help='Test to run. Can be specified as one or more files or ' \
            'directories, or omitted. If omitted, the entire test suite is ' \
            'executed.')

    def run_cppunit_test(self, **params):
        import runcppunittests as cppunittests
        import logging

        if len(params['test_files']) == 0:
            testdir = os.path.join(self.distdir, 'cppunittests')
            progs = cppunittests.extract_unittests_from_args([testdir], None)
        else:
            progs = cppunittests.extract_unittests_from_args(params['test_files'], None)

        # See if we have crash symbols
        symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')
        if not os.path.isdir(symbols_path):
            symbols_path = None

        tester = cppunittests.CPPUnitTests()
        try:
            result = tester.run_tests(progs, self.bindir, symbols_path)
        except Exception, e:
            self.log(logging.ERROR, 'cppunittests',
                {'exception': str(e)},
                'Caught exception running cpp unit tests: {exception}')
            result = False

        return 0 if result else 1

@CommandProvider
class CheckSpiderMonkeyCommand(MachCommandBase):
    @Command('check-spidermonkey', category='testing', description='Run SpiderMonkey tests.')
    @CommandArgument('--valgrind', action='store_true', help='Run jit-test suite with valgrind flag')

    def run_checkspidermonkey(self, **params):
        import subprocess
        import sys

        bin_suffix = ''
        if sys.platform.startswith('win'):
            bin_suffix = '.exe'

        js = os.path.join(self.bindir, 'js%s' % bin_suffix)

        print('Running jit-tests')
        jittest_cmd = [os.path.join(self.topsrcdir, 'js', 'src', 'jit-test', 'jit_test.py'),
              js, '--no-slow', '--tbpl']
        if params['valgrind']:
            jittest_cmd.append('--valgrind')

        jittest_result = subprocess.call(jittest_cmd)

        print('running jstests')
        jstest_cmd = [os.path.join(self.topsrcdir, 'js', 'src', 'tests', 'jstests.py'),
              js, '--tbpl']
        jstest_result = subprocess.call(jstest_cmd)

        print('running jsapi-tests')
        jsapi_tests_cmd = [os.path.join(self.bindir, 'jsapi-tests%s' % bin_suffix)]
        jsapi_tests_result = subprocess.call(jsapi_tests_cmd)

        print('running check-style')
        check_style_cmd = [sys.executable, os.path.join(self.topsrcdir, 'config', 'check_spidermonkey_style.py')]
        check_style_result = subprocess.call(check_style_cmd, cwd=os.path.join(self.topsrcdir, 'js', 'src'))

        all_passed = jittest_result and jstest_result and jsapi_tests_result and check_style_result

        return all_passed