testing/tools/mach_test_package_bootstrap.py
author Cosmin Sabou <csabou@mozilla.com>
Sat, 03 Nov 2018 03:12:43 +0200
changeset 444308 63f135d2db22375b51cf0ded745279afe2188022
parent 398487 b015e89b1064b5f0958ad2b945c9b2fdbe18f48f
child 462864 975644295f21a5bd55d551abd09109a2b7d771eb
permissions -rw-r--r--
Backed out 48 changesets (bug 1376873, bug 1502992) for causing windows 7 wpt crashes on several files. Backed out changeset b9db1780bb9c (bug 1502992) Backed out changeset f9f879bd247c (bug 1376873) Backed out changeset f029e9f3ceaf (bug 1376873) Backed out changeset 7317f657a9a7 (bug 1376873) Backed out changeset 4efbe4d0050c (bug 1376873) Backed out changeset 26e09a6dee05 (bug 1376873) Backed out changeset afbd3a8108de (bug 1376873) Backed out changeset 23b5fb1a2577 (bug 1376873) Backed out changeset 5d9c7a09616c (bug 1376873) Backed out changeset f15c4be0f19f (bug 1376873) Backed out changeset 3d11b041baf2 (bug 1376873) Backed out changeset 9674bbefc68e (bug 1376873) Backed out changeset 5a45d2c706d6 (bug 1376873) Backed out changeset 9b82f16618b9 (bug 1376873) Backed out changeset 0c3c2fc4646b (bug 1376873) Backed out changeset 381df4c6ae88 (bug 1376873) Backed out changeset acc06acb5342 (bug 1376873) Backed out changeset 60fdf3598932 (bug 1376873) Backed out changeset 176b82dccf4c (bug 1376873) Backed out changeset 58e47be76761 (bug 1376873) Backed out changeset d082eb8d80c6 (bug 1376873) Backed out changeset 622f78404b8a (bug 1376873) Backed out changeset 086c722131f9 (bug 1376873) Backed out changeset 79010785539c (bug 1376873) Backed out changeset 6b2919ed789b (bug 1376873) Backed out changeset da8d3eb163bd (bug 1376873) Backed out changeset 0fc8c65e163e (bug 1376873) Backed out changeset de3f5bd6846b (bug 1376873) Backed out changeset 2951112ade46 (bug 1376873) Backed out changeset 9fe47f8ea1b8 (bug 1376873) Backed out changeset b94935661622 (bug 1376873) Backed out changeset 6eca45c83131 (bug 1376873) Backed out changeset 909f472c2e40 (bug 1376873) Backed out changeset 19d9a30b300c (bug 1376873) Backed out changeset 6a52b92ff037 (bug 1376873) Backed out changeset 6a6771656fe5 (bug 1376873) Backed out changeset ffdbeba93fa6 (bug 1376873) Backed out changeset d12ad321047d (bug 1376873) Backed out changeset d1f7d44e05a4 (bug 1376873) Backed out changeset b59e1fca5c13 (bug 1376873) Backed out changeset 5d7a8864505d (bug 1376873) Backed out changeset e1ea09e84feb (bug 1376873) Backed out changeset bc92eac0522f (bug 1376873) Backed out changeset 1f86d23e4806 (bug 1376873) Backed out changeset d78c5c50b6b8 (bug 1376873) Backed out changeset 2e1af73cdd2e (bug 1376873) Backed out changeset dda3c784b0c0 (bug 1376873) Backed out changeset f40ca921b74b (bug 1376873)

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import print_function, unicode_literals

import json
import os
import platform
import sys
import types


SEARCH_PATHS = [
    'marionette/harness',
    'marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py',
    'marionette/client',
    'mochitest',
    'mozbase/manifestparser',
    'mozbase/mozcrash',
    'mozbase/mozdebug',
    'mozbase/mozdevice',
    'mozbase/mozfile',
    'mozbase/mozhttpd',
    'mozbase/mozinfo',
    'mozbase/mozinstall',
    'mozbase/mozleak',
    'mozbase/mozlog',
    'mozbase/moznetwork',
    'mozbase/mozprocess',
    'mozbase/mozprofile',
    'mozbase/mozrunner',
    'mozbase/mozscreenshot',
    'mozbase/mozsystemmonitor',
    'mozbase/moztest',
    'mozbase/mozversion',
    'reftest',
    'tools/mach',
    'tools/mozterm',
    'tools/six',
    'tools/wptserve',
    'web-platform',
    'web-platform/tests/tools/wptrunner',
    'xpcshell',
]

# Individual files providing mach commands.
MACH_MODULES = [
    'marionette/mach_test_package_commands.py',
    'mochitest/mach_test_package_commands.py',
    'reftest/mach_test_package_commands.py',
    'tools/mach/mach/commands/commandinfo.py',
    'web-platform/mach_test_package_commands.py',
    'xpcshell/mach_test_package_commands.py',
]


CATEGORIES = {
    'testing': {
        'short': 'Testing',
        'long': 'Run tests.',
        'priority': 30,
    },
    'devenv': {
        'short': 'Development Environment',
        'long': 'Set up and configure your development environment.',
        'priority': 20,
    },
    'misc': {
        'short': 'Potpourri',
        'long': 'Potent potables and assorted snacks.',
        'priority': 10,
    },
    'disabled': {
        'short': 'Disabled',
        'long': 'The disabled commands are hidden by default. Use -v to display them. '
                'These commands are unavailable for your current context, '
                'run "mach <command>" to see why.',
        'priority': 0,
    }
}


IS_WIN = sys.platform in ('win32', 'cygwin')


def ancestors(path, depth=0):
    """Emit the parent directories of a path."""
    count = 1
    while path and count != depth:
        yield path
        newpath = os.path.dirname(path)
        if newpath == path:
            break
        path = newpath
        count += 1


def activate_mozharness_venv(context):
    """Activate the mozharness virtualenv in-process."""
    venv = os.path.join(context.mozharness_workdir,
                        context.mozharness_config.get('virtualenv_path', 'venv'))

    if not os.path.isdir(venv):
        print("No mozharness virtualenv detected at '{}'.".format(venv))
        return 1

    venv_bin = os.path.join(venv, 'Scripts' if IS_WIN else 'bin')
    activate_path = os.path.join(venv_bin, 'activate_this.py')

    execfile(activate_path, dict(__file__=activate_path))

    if isinstance(os.environ['PATH'], unicode):
        os.environ['PATH'] = os.environ['PATH'].encode('utf-8')

    # sys.executable is used by mochitest-media to start the websocketprocessbridge,
    # for some reason it doesn't get set when calling `activate_this.py` so set it
    # here instead.
    binary = 'python'
    if IS_WIN:
        binary += '.exe'
    sys.executable = os.path.join(venv_bin, binary)


def find_firefox(context):
    """Try to automagically find the firefox binary."""
    import mozinstall
    search_paths = []

    # Check for a mozharness setup
    config = context.mozharness_config
    if config and 'binary_path' in config:
        return config['binary_path']
    elif config:
        search_paths.append(os.path.join(context.mozharness_workdir, 'application'))

    # Check for test-stage setup
    dist_bin = os.path.join(os.path.dirname(context.package_root), 'bin')
    if os.path.isdir(dist_bin):
        search_paths.append(dist_bin)

    for path in search_paths:
        try:
            return mozinstall.get_binary(path, 'firefox')
        except mozinstall.InvalidBinary:
            continue


def find_hostutils(context):
    workdir = context.mozharness_workdir
    hostutils = os.path.join(workdir, 'hostutils')
    for fname in os.listdir(hostutils):
        fpath = os.path.join(hostutils, fname)
        if os.path.isdir(fpath) and fname.startswith('host-utils'):
            return fpath


def normalize_test_path(test_root, path):
    if os.path.isabs(path) or os.path.exists(path):
        return os.path.normpath(os.path.abspath(path))

    for parent in ancestors(test_root):
        test_path = os.path.join(parent, path)
        if os.path.exists(test_path):
            return os.path.normpath(os.path.abspath(test_path))
    # Not a valid path? Return as is and let test harness deal with it
    return path


def bootstrap(test_package_root):
    test_package_root = os.path.abspath(test_package_root)

    # Ensure we are running Python 2.7+. We put this check here so we generate a
    # user-friendly error message rather than a cryptic stack trace on module
    # import.
    if sys.version_info[0] != 2 or sys.version_info[1] < 7:
        print('Python 2.7 or above (but not Python 3) is required to run mach.')
        print('You are running Python', platform.python_version())
        sys.exit(1)

    sys.path[0:0] = [os.path.join(test_package_root, path) for path in SEARCH_PATHS]
    import mach.main

    def populate_context(context, key=None):
        if key is None:
            context.package_root = test_package_root
            context.bin_dir = os.path.join(test_package_root, 'bin')
            context.certs_dir = os.path.join(test_package_root, 'certs')
            context.module_dir = os.path.join(test_package_root, 'modules')
            context.ancestors = ancestors
            context.normalize_test_path = normalize_test_path
            return

        # The values for the following 'key's will be set lazily, and cached
        # after first being invoked.
        if key == 'firefox_bin':
            return find_firefox(context)

        if key == 'hostutils':
            return find_hostutils(context)

        if key == 'mozharness_config':
            for dir_path in ancestors(context.package_root):
                mozharness_config = os.path.join(dir_path, 'logs', 'localconfig.json')
                if os.path.isfile(mozharness_config):
                    with open(mozharness_config, 'rb') as f:
                        return json.load(f)
            return {}

        if key == 'mozharness_workdir':
            config = context.mozharness_config
            if config:
                return os.path.join(config['base_work_dir'], config['work_dir'])

        if key == 'activate_mozharness_venv':
            return types.MethodType(activate_mozharness_venv, context)

    mach = mach.main.Mach(os.getcwd())
    mach.populate_context_handler = populate_context

    for category, meta in CATEGORIES.items():
        mach.define_category(category, meta['short'], meta['long'],
                             meta['priority'])

    for path in MACH_MODULES:
        cmdfile = os.path.join(test_package_root, path)

        # Depending on which test zips were extracted,
        # the command module might not exist
        if os.path.isfile(cmdfile):
            mach.load_commands_from_file(cmdfile)

    return mach