python/mozbuild/mozbuild/frontend/data.py
author Nathan Froyd <froydnj@mozilla.com>
Thu, 14 Nov 2019 11:16:38 -0400
changeset 501561 d64119136e15458d4aea5c548b4d10d7099d78f6
parent 494559 69a6bb18996873f9a07080bc1d800ad2f1014d96
permissions -rw-r--r--
Bug 1579189 - raise the minimum clang version to 5; r=#build We need this for "full" C++17 support (everything is supported, but some C++17 features still have bugs) and this change also brings Linux into parity with our Mac requirements. MANUAL PUSH: build toolchains on inbound to avoid clogging autoland Differential Revision: https://phabricator.services.mozilla.com/D51450

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

r"""Data structures representing Mozilla's source tree.

The frontend files are parsed into static data structures. These data
structures are defined in this module.

All data structures of interest are children of the TreeMetadata class.

Logic for populating these data structures is not defined in this class.
Instead, what we have here are dumb container classes. The emitter module
contains the code for converting executed mozbuild files into these data
structures.
"""

from __future__ import absolute_import, print_function, unicode_literals

from mozbuild.frontend.context import (
    ObjDirPath,
    SourcePath,
)
from mozpack.chrome.manifest import ManifestEntry

import mozpack.path as mozpath
from .context import FinalTargetValue

from collections import defaultdict
import itertools

from ..util import (
    group_unified_files,
)

from ..testing import (
    all_test_flavors,
)


class TreeMetadata(object):
    """Base class for all data being captured."""
    __slots__ = ()

    def to_dict(self):
        return {k.lower(): getattr(self, k) for k in self.DICT_ATTRS}


class ContextDerived(TreeMetadata):
    """Build object derived from a single Context instance.

    It holds fields common to all context derived classes. This class is likely
    never instantiated directly but is instead derived from.
    """

    __slots__ = (
        'context_main_path',
        'context_all_paths',
        'topsrcdir',
        'topobjdir',
        'relsrcdir',
        'srcdir',
        'objdir',
        'config',
        '_context',
    )

    def __init__(self, context):
        TreeMetadata.__init__(self)

        # Capture the files that were evaluated to fill this context.
        self.context_main_path = context.main_path
        self.context_all_paths = context.all_paths

        # Basic directory state.
        self.topsrcdir = context.config.topsrcdir
        self.topobjdir = context.config.topobjdir

        self.relsrcdir = context.relsrcdir
        self.srcdir = context.srcdir
        self.objdir = context.objdir

        self.config = context.config

        self._context = context

    @property
    def install_target(self):
        return self._context['FINAL_TARGET']

    @property
    def installed(self):
        return self._context['DIST_INSTALL'] is not False

    @property
    def defines(self):
        defines = self._context['DEFINES']
        return Defines(self._context, defines) if defines else None

    @property
    def relobjdir(self):
        return mozpath.relpath(self.objdir, self.topobjdir)


class HostMixin(object):
    @property
    def defines(self):
        defines = self._context['HOST_DEFINES']
        return HostDefines(self._context, defines) if defines else None


class DirectoryTraversal(ContextDerived):
    """Describes how directory traversal for building should work.

    This build object is likely only of interest to the recursive make backend.
    Other build backends should (ideally) not attempt to mimic the behavior of
    the recursive make backend. The only reason this exists is to support the
    existing recursive make backend while the transition to mozbuild frontend
    files is complete and we move to a more optimal build backend.

    Fields in this class correspond to similarly named variables in the
    frontend files.
    """
    __slots__ = (
        'dirs',
    )

    def __init__(self, context):
        ContextDerived.__init__(self, context)

        self.dirs = []


class BaseConfigSubstitution(ContextDerived):
    """Base class describing autogenerated files as part of config.status."""

    __slots__ = (
        'input_path',
        'output_path',
        'relpath',
    )

    def __init__(self, context):
        ContextDerived.__init__(self, context)

        self.input_path = None
        self.output_path = None
        self.relpath = None


class ConfigFileSubstitution(BaseConfigSubstitution):
    """Describes a config file that will be generated using substitutions."""


class VariablePassthru(ContextDerived):
    """A dict of variables to pass through to backend.mk unaltered.

    The purpose of this object is to facilitate rapid transitioning of
    variables from Makefile.in to moz.build. In the ideal world, this class
    does not exist and every variable has a richer class representing it.
    As long as we rely on this class, we lose the ability to have flexibility
    in our build backends since we will continue to be tied to our rules.mk.
    """
    __slots__ = ('variables')

    def __init__(self, context):
        ContextDerived.__init__(self, context)
        self.variables = {}


class ComputedFlags(ContextDerived):
    """Aggregate flags for consumption by various backends.
    """
    __slots__ = ('flags',)

    def __init__(self, context, reader_flags):
        ContextDerived.__init__(self, context)
        self.flags = reader_flags

    def resolve_flags(self, key, value):
        # Bypass checks done by CompileFlags that would keep us from
        # setting a value here.
        dict.__setitem__(self.flags, key, value)

    def get_flags(self):
        flags = defaultdict(list)
        for key, _, dest_vars in self.flags.flag_variables:
            value = self.flags.get(key)
            if value:
                for dest_var in dest_vars:
                    flags[dest_var].extend(value)
        return flags.items()


class XPIDLModule(ContextDerived):
    """Describes an XPIDL module to be compiled."""

    __slots__ = (
        'name',
        'idl_files',
    )

    def __init__(self, context, name, idl_files):
        ContextDerived.__init__(self, context)

        assert all(isinstance(idl, SourcePath) for idl in idl_files)
        self.name = name
        self.idl_files = idl_files


class BaseDefines(ContextDerived):
    """Context derived container object for DEFINES/HOST_DEFINES,
    which are OrderedDicts.
    """
    __slots__ = ('defines')

    def __init__(self, context, defines):
        ContextDerived.__init__(self, context)
        self.defines = defines

    def get_defines(self):
        for define, value in self.defines.iteritems():
            if value is True:
                yield('-D%s' % define)
            elif value is False:
                yield('-U%s' % define)
            else:
                yield('-D%s=%s' % (define, value))

    def update(self, more_defines):
        if isinstance(more_defines, Defines):
            self.defines.update(more_defines.defines)
        else:
            self.defines.update(more_defines)


class Defines(BaseDefines):
    pass


class HostDefines(BaseDefines):
    pass


class WebIDLCollection(ContextDerived):
    """Collects WebIDL info referenced during the build."""

    def __init__(self, context):
        ContextDerived.__init__(self, context)
        self.sources = set()
        self.generated_sources = set()
        self.generated_events_sources = set()
        self.preprocessed_sources = set()
        self.test_sources = set()
        self.preprocessed_test_sources = set()
        self.example_interfaces = set()

    def all_regular_sources(self):
        return self.sources | self.generated_sources | \
            self.generated_events_sources | self.preprocessed_sources

    def all_regular_basenames(self):
        return [mozpath.basename(source) for source in self.all_regular_sources()]

    def all_regular_stems(self):
        return [mozpath.splitext(b)[0] for b in self.all_regular_basenames()]

    def all_regular_bindinggen_stems(self):
        for stem in self.all_regular_stems():
            yield '%sBinding' % stem

        for source in self.generated_events_sources:
            yield mozpath.splitext(mozpath.basename(source))[0]

    def all_regular_cpp_basenames(self):
        for stem in self.all_regular_bindinggen_stems():
            yield '%s.cpp' % stem

    def all_test_sources(self):
        return self.test_sources | self.preprocessed_test_sources

    def all_test_basenames(self):
        return [mozpath.basename(source) for source in self.all_test_sources()]

    def all_test_stems(self):
        return [mozpath.splitext(b)[0] for b in self.all_test_basenames()]

    def all_test_cpp_basenames(self):
        return sorted('%sBinding.cpp' % s for s in self.all_test_stems())

    def all_static_sources(self):
        return self.sources | self.generated_events_sources | \
            self.test_sources

    def all_non_static_sources(self):
        return self.generated_sources | self.all_preprocessed_sources()

    def all_non_static_basenames(self):
        return [mozpath.basename(s) for s in self.all_non_static_sources()]

    def all_preprocessed_sources(self):
        return self.preprocessed_sources | self.preprocessed_test_sources

    def all_sources(self):
        return set(self.all_regular_sources()) | set(self.all_test_sources())

    def all_basenames(self):
        return [mozpath.basename(source) for source in self.all_sources()]

    def all_stems(self):
        return [mozpath.splitext(b)[0] for b in self.all_basenames()]

    def generated_events_basenames(self):
        return [mozpath.basename(s) for s in self.generated_events_sources]

    def generated_events_stems(self):
        return [mozpath.splitext(b)[0] for b in self.generated_events_basenames()]

    @property
    def unified_source_mapping(self):
        # Bindings are compiled in unified mode to speed up compilation and
        # to reduce linker memory size. Note that test bindings are separated
        # from regular ones so tests bindings aren't shipped.
        return list(group_unified_files(self.all_regular_cpp_basenames(),
                                        unified_prefix='UnifiedBindings',
                                        unified_suffix='cpp',
                                        files_per_unified_file=32))

    def all_source_files(self):
        from mozwebidlcodegen import WebIDLCodegenManager
        return (sorted(list(WebIDLCodegenManager.GLOBAL_DEFINE_FILES)) +
                sorted(set(p for p, _ in self.unified_source_mapping)))


class IPDLCollection(ContextDerived):
    """Collects IPDL files during the build."""

    def __init__(self, context):
        ContextDerived.__init__(self, context)
        self.sources = set()
        self.preprocessed_sources = set()

    def all_sources(self):
        return self.sources | self.preprocessed_sources

    def all_regular_sources(self):
        return self.sources

    def all_preprocessed_sources(self):
        return self.preprocessed_sources

    def all_generated_sources(self):
        sorted_ipdl_sources = list(sorted(self.all_sources()))

        def files_from(ipdl):
            base = mozpath.basename(ipdl)
            root, ext = mozpath.splitext(base)

            # Both .ipdl and .ipdlh become .cpp files
            files = ['%s.cpp' % root]
            if ext == '.ipdl':
                # .ipdl also becomes Child/Parent.cpp files
                files.extend(['%sChild.cpp' % root,
                              '%sParent.cpp' % root])
            return files

        return list(itertools.chain(*[files_from(p) for p in sorted_ipdl_sources]))

    @property
    def unified_source_mapping(self):
        return list(group_unified_files(self.all_generated_sources(),
                                        unified_prefix='UnifiedProtocols',
                                        unified_suffix='cpp',
                                        files_per_unified_file=16))

    def all_source_files(self):
        return sorted(set(p for p, _ in self.unified_source_mapping))


class XPCOMComponentManifests(ContextDerived):
    """Collects XPCOM manifest files during the build."""

    def __init__(self, context):
        ContextDerived.__init__(self, context)
        self.manifests = set()

    def all_sources(self):
        return self.manifests

    def all_source_files(self):
        return []


class LinkageWrongKindError(Exception):
    """Error thrown when trying to link objects of the wrong kind"""


class Linkable(ContextDerived):
    """Generic context derived container object for programs and libraries"""
    __slots__ = (
        'cxx_link',
        'lib_defines',
        'linked_libraries',
        'linked_system_libs',
        'no_pgo_sources',
        'no_pgo',
        'sources',
    )

    def __init__(self, context):
        ContextDerived.__init__(self, context)
        self.cxx_link = False
        self.linked_libraries = []
        self.linked_system_libs = []
        self.lib_defines = Defines(context, {})
        self.sources = defaultdict(list)
        self.no_pgo_sources = []
        self.no_pgo = False

    def link_library(self, obj):
        assert isinstance(obj, BaseLibrary)
        if obj.KIND != self.KIND:
            raise LinkageWrongKindError('%s != %s' % (obj.KIND, self.KIND))
        self.linked_libraries.append(obj)
        if obj.cxx_link and not isinstance(obj, SharedLibrary):
            self.cxx_link = True
        obj.refs.append(self)

    def link_system_library(self, lib):
        # The '$' check is here as a special temporary rule, allowing the
        # inherited use of make variables, most notably in TK_LIBS.
        if not lib.startswith('$') and not lib.startswith('-'):
            type_var = 'HOST_CC_TYPE' if self.KIND == 'host' else 'CC_TYPE'
            compiler_type = self.config.substs.get(type_var)
            if compiler_type in ('gcc', 'clang'):
                lib = '-l%s' % lib
            else:
                lib = '%s%s%s' % (
                    self.config.import_prefix,
                    lib,
                    self.config.import_suffix,
                )
        self.linked_system_libs.append(lib)

    def source_files(self):
        all_sources = []
        # This is ordered for reproducibility and consistently w/
        # config/rules.mk
        for suffix in ('.c', '.S', '.cpp', '.m', '.mm', '.s'):
            all_sources += self.sources.get(suffix, [])
        return all_sources

    def _get_objs(self, sources):
        obj_prefix = ''
        if self.KIND == 'host':
            obj_prefix = 'host_'

        return [mozpath.join(self.objdir, '%s%s.%s' % (obj_prefix,
                                                       mozpath.splitext(mozpath.basename(f))[0],
                                                       self.config.substs.get('OBJ_SUFFIX', '')))
                for f in sources]

    @property
    def no_pgo_objs(self):
        return self._get_objs(self.no_pgo_sources)

    @property
    def objs(self):
        return self._get_objs(self.source_files())


class BaseProgram(Linkable):
    """Context derived container object for programs, which is a unicode
    string.

    This class handles automatically appending a binary suffix to the program
    name.
    If the suffix is not defined, the program name is unchanged.
    Otherwise, if the program name ends with the given suffix, it is unchanged
    Otherwise, the suffix is appended to the program name.
    """
    __slots__ = ('program')

    DICT_ATTRS = {
        'install_target',
        'KIND',
        'program',
        'relobjdir',
    }

    def __init__(self, context, program, is_unit_test=False):
        Linkable.__init__(self, context)

        bin_suffix = context.config.substs.get(self.SUFFIX_VAR, '')
        if not program.endswith(bin_suffix):
            program += bin_suffix
        self.program = program
        self.is_unit_test = is_unit_test

    @property
    def output_path(self):
        if self.installed:
            return ObjDirPath(self._context, '!/' + mozpath.join(
                self.install_target, self.program))
        else:
            return ObjDirPath(self._context, '!' + self.program)

    def __repr__(self):
        return '<%s: %s/%s>' % (type(self).__name__, self.relobjdir, self.program)

    @property
    def name(self):
        return self.program


class Program(BaseProgram):
    """Context derived container object for PROGRAM"""
    SUFFIX_VAR = 'BIN_SUFFIX'
    KIND = 'target'


class HostProgram(HostMixin, BaseProgram):
    """Context derived container object for HOST_PROGRAM"""
    SUFFIX_VAR = 'HOST_BIN_SUFFIX'
    KIND = 'host'

    @property
    def install_target(self):
        return 'dist/host/bin'


class SimpleProgram(BaseProgram):
    """Context derived container object for each program in SIMPLE_PROGRAMS"""
    SUFFIX_VAR = 'BIN_SUFFIX'
    KIND = 'target'

    def source_files(self):
        for srcs in self.sources.values():
            for f in srcs:
                if mozpath.basename(mozpath.splitext(f)[0]) == mozpath.splitext(self.program)[0]:
                    return [f]
        return []


class HostSimpleProgram(HostMixin, BaseProgram):
    """Context derived container object for each program in
    HOST_SIMPLE_PROGRAMS"""
    SUFFIX_VAR = 'HOST_BIN_SUFFIX'
    KIND = 'host'

    def source_files(self):
        for srcs in self.sources.values():
            for f in srcs:
                if ('host_%s' % mozpath.basename(mozpath.splitext(f)[0]) ==
                    mozpath.splitext(self.program)[0]):
                    return [f]
        return []


def cargo_output_directory(context, target_var):
    # cargo creates several directories and places its build artifacts
    # in those directories.  The directory structure depends not only
    # on the target, but also what sort of build we are doing.
    rust_build_kind = 'release'
    if context.config.substs.get('MOZ_DEBUG_RUST'):
        rust_build_kind = 'debug'
    return mozpath.join(context.config.substs[target_var], rust_build_kind)


# Rust programs aren't really Linkable, since Cargo handles all the details
# of linking things.
class BaseRustProgram(ContextDerived):
    __slots__ = (
        'name',
        'cargo_file',
        'location',
        'SUFFIX_VAR',
        'KIND',
        'TARGET_SUBST_VAR',
    )

    def __init__(self, context, name, cargo_file):
        ContextDerived.__init__(self, context)
        self.name = name
        self.cargo_file = cargo_file
        cargo_dir = cargo_output_directory(context, self.TARGET_SUBST_VAR)
        exe_file = '%s%s' % (name, context.config.substs.get(self.SUFFIX_VAR, ''))
        self.location = mozpath.join(cargo_dir, exe_file)


class RustProgram(BaseRustProgram):
    SUFFIX_VAR = 'BIN_SUFFIX'
    KIND = 'target'
    TARGET_SUBST_VAR = 'RUST_TARGET'


class HostRustProgram(BaseRustProgram):
    SUFFIX_VAR = 'HOST_BIN_SUFFIX'
    KIND = 'host'
    TARGET_SUBST_VAR = 'RUST_HOST_TARGET'


class RustTests(ContextDerived):
    __slots__ = (
        'names',
        'features',
        'output_category',
    )

    def __init__(self, context, names, features):
        ContextDerived.__init__(self, context)
        self.names = names
        self.features = features
        self.output_category = 'rusttests'


class BaseLibrary(Linkable):
    """Generic context derived container object for libraries."""
    __slots__ = (
        'basename',
        'lib_name',
        'import_name',
        'refs',
    )

    def __init__(self, context, basename):
        Linkable.__init__(self, context)

        self.basename = self.lib_name = basename
        if self.lib_name:
            self.lib_name = '%s%s%s' % (
                context.config.lib_prefix,
                self.lib_name,
                context.config.lib_suffix
            )
            self.import_name = self.lib_name

        self.refs = []

    def __repr__(self):
        return '<%s: %s/%s>' % (type(self).__name__, self.relobjdir, self.lib_name)

    @property
    def name(self):
        return self.lib_name


class Library(BaseLibrary):
    """Context derived container object for a library"""
    KIND = 'target'
    __slots__ = (
    )

    def __init__(self, context, basename, real_name=None):
        BaseLibrary.__init__(self, context, real_name or basename)
        self.basename = basename


class StaticLibrary(Library):
    """Context derived container object for a static library"""
    __slots__ = (
        'link_into',
        'no_expand_lib',
    )

    def __init__(self, context, basename, real_name=None,
                 link_into=None, no_expand_lib=False):
        Library.__init__(self, context, basename, real_name)
        self.link_into = link_into
        self.no_expand_lib = no_expand_lib


class BaseRustLibrary(object):
    slots = (
        'cargo_file',
        'crate_type',
        'dependencies',
        'deps_path',
        'features',
        'target_dir',
        'output_category',
    )

    def init(self, context, basename, cargo_file, crate_type, dependencies,
             features, target_dir):
        self.cargo_file = cargo_file
        self.crate_type = crate_type
        # We need to adjust our naming here because cargo replaces '-' in
        # package names defined in Cargo.toml with underscores in actual
        # filenames. But we need to keep the basename consistent because
        # many other things in the build system depend on that.
        assert self.crate_type == 'staticlib'
        self.lib_name = '%s%s%s' % (context.config.rust_lib_prefix,
                                    basename.replace('-', '_'),
                                    context.config.rust_lib_suffix)
        self.dependencies = dependencies
        self.features = features
        self.target_dir = target_dir
        self.output_category = context.get('RUST_LIBRARY_OUTPUT_CATEGORY')
        # Skip setting properties below which depend on cargo
        # when we don't have a compile environment. The required
        # config keys won't be available, but the instance variables
        # that we don't set should never be accessed by the actual
        # build in that case.
        if not context.config.substs.get('COMPILE_ENVIRONMENT'):
            return
        build_dir = mozpath.join(target_dir,
                                 cargo_output_directory(context, self.TARGET_SUBST_VAR))
        self.import_name = mozpath.join(build_dir, self.lib_name)
        self.deps_path = mozpath.join(build_dir, 'deps')


class RustLibrary(StaticLibrary, BaseRustLibrary):
    """Context derived container object for a rust static library"""
    KIND = 'target'
    TARGET_SUBST_VAR = 'RUST_TARGET'
    FEATURES_VAR = 'RUST_LIBRARY_FEATURES'
    LIB_FILE_VAR = 'RUST_LIBRARY_FILE'
    __slots__ = BaseRustLibrary.slots

    def __init__(self, context, basename, cargo_file, crate_type, dependencies,
                 features, target_dir, link_into=None):
        StaticLibrary.__init__(self, context, basename, link_into=link_into,
                               # A rust library is a real static library ; make
                               # it known to the build system.
                               no_expand_lib=True)
        BaseRustLibrary.init(self, context, basename, cargo_file,
                             crate_type, dependencies, features, target_dir)


class SharedLibrary(Library):
    """Context derived container object for a shared library"""
    __slots__ = (
        'soname',
        'variant',
        'symbols_file',
        'output_category',
        'symbols_link_arg',
    )

    DICT_ATTRS = {
        'basename',
        'import_name',
        'install_target',
        'lib_name',
        'relobjdir',
        'soname',
    }

    FRAMEWORK = 1
    MAX_VARIANT = 2

    def __init__(self, context, basename, real_name=None,
                 soname=None, variant=None, symbols_file=False):
        assert(variant in range(1, self.MAX_VARIANT) or variant is None)
        Library.__init__(self, context, basename, real_name)
        self.variant = variant
        self.lib_name = real_name or basename
        self.output_category = context.get('SHARED_LIBRARY_OUTPUT_CATEGORY')
        assert self.lib_name

        if variant == self.FRAMEWORK:
            self.import_name = self.lib_name
        else:
            self.import_name = '%s%s%s' % (
                context.config.import_prefix,
                self.lib_name,
                context.config.import_suffix,
            )
            self.lib_name = '%s%s%s' % (
                context.config.dll_prefix,
                self.lib_name,
                context.config.dll_suffix,
            )
        if soname:
            self.soname = '%s%s%s' % (
                context.config.dll_prefix,
                soname,
                context.config.dll_suffix,
            )
        else:
            self.soname = self.lib_name

        if symbols_file is False:
            # No symbols file.
            self.symbols_file = None
        elif symbols_file is True:
            # Symbols file with default name.
            if context.config.substs['OS_TARGET'] == 'WINNT':
                self.symbols_file = '%s.def' % self.lib_name
            else:
                self.symbols_file = '%s.symbols' % self.lib_name
        else:
            # Explicitly provided name.
            self.symbols_file = symbols_file

        if self.symbols_file:
            os_target = context.config.substs['OS_TARGET']
            if os_target == 'Darwin':
                self.symbols_link_arg = '-Wl,-exported_symbols_list,' + self.symbols_file
            elif os_target == 'WINNT':
                if context.config.substs.get('GNU_CC'):
                    self.symbols_link_arg = self.symbols_file
                else:
                    self.symbols_link_arg = '-DEF:' + self.symbols_file
            elif context.config.substs.get('GCC_USE_GNU_LD'):
                self.symbols_link_arg = '-Wl,--version-script,' + self.symbols_file


class HostSharedLibrary(HostMixin, Library):
    """Context derived container object for a host shared library.

    This class supports less things than SharedLibrary does for target shared
    libraries. Currently has enough build system support to build the clang
    plugin."""
    KIND = 'host'

    def __init__(self, context, basename):
        Library.__init__(self, context, basename)
        self.lib_name = '%s%s%s' % (
            context.config.host_dll_prefix,
            self.basename,
            context.config.host_dll_suffix,
        )


class ExternalLibrary(object):
    """Empty mixin for libraries built by an external build system."""


class ExternalStaticLibrary(StaticLibrary, ExternalLibrary):
    """Context derived container for static libraries built by an external
    build system."""


class ExternalSharedLibrary(SharedLibrary, ExternalLibrary):
    """Context derived container for shared libraries built by an external
    build system."""


class HostLibrary(HostMixin, BaseLibrary):
    """Context derived container object for a host library"""
    KIND = 'host'
    no_expand_lib = False


class HostRustLibrary(HostLibrary, BaseRustLibrary):
    """Context derived container object for a host rust library"""
    KIND = 'host'
    TARGET_SUBST_VAR = 'RUST_HOST_TARGET'
    FEATURES_VAR = 'HOST_RUST_LIBRARY_FEATURES'
    LIB_FILE_VAR = 'HOST_RUST_LIBRARY_FILE'
    __slots__ = BaseRustLibrary.slots
    no_expand_lib = True

    def __init__(self, context, basename, cargo_file, crate_type, dependencies,
                 features, target_dir):
        HostLibrary.__init__(self, context, basename)
        BaseRustLibrary.init(self, context, basename, cargo_file,
                             crate_type, dependencies, features, target_dir)


class TestManifest(ContextDerived):
    """Represents a manifest file containing information about tests."""

    __slots__ = (
        # The type of test manifest this is.
        'flavor',

        # Maps source filename to destination filename. The destination
        # path is relative from the tests root directory. Values are 2-tuples
        # of (destpath, is_test_file) where the 2nd item is True if this
        # item represents a test file (versus a support file).
        'installs',

        # A list of pattern matching installs to perform. Entries are
        # (base, pattern, dest).
        'pattern_installs',

        # Where all files for this manifest flavor are installed in the unified
        # test package directory.
        'install_prefix',

        # Set of files provided by an external mechanism.
        'external_installs',

        # Set of files required by multiple test directories, whose installation
        # will be resolved when running tests.
        'deferred_installs',

        # The full path of this manifest file.
        'path',

        # The directory where this manifest is defined.
        'directory',

        # The parsed manifestparser.TestManifest instance.
        'manifest',

        # List of tests. Each element is a dict of metadata.
        'tests',

        # The relative path of the parsed manifest within the srcdir.
        'manifest_relpath',

        # The relative path of the parsed manifest within the objdir.
        'manifest_obj_relpath',

        # The relative paths to all source files for this manifest.
        'source_relpaths',

        # If this manifest is a duplicate of another one, this is the
        # manifestparser.TestManifest of the other one.
        'dupe_manifest',
    )

    def __init__(self, context, path, manifest, flavor=None,
                 install_prefix=None, relpath=None, sources=(),
                 dupe_manifest=False):
        ContextDerived.__init__(self, context)

        assert flavor in all_test_flavors()

        self.path = path
        self.directory = mozpath.dirname(path)
        self.manifest = manifest
        self.flavor = flavor
        self.install_prefix = install_prefix
        self.manifest_relpath = relpath
        self.manifest_obj_relpath = relpath
        self.source_relpaths = sources
        self.dupe_manifest = dupe_manifest
        self.installs = {}
        self.pattern_installs = []
        self.tests = []
        self.external_installs = set()
        self.deferred_installs = set()


class LocalInclude(ContextDerived):
    """Describes an individual local include path."""

    __slots__ = (
        'path',
    )

    def __init__(self, context, path):
        ContextDerived.__init__(self, context)

        self.path = path


class PerSourceFlag(ContextDerived):
    """Describes compiler flags specified for individual source files."""

    __slots__ = (
        'file_name',
        'flags',
    )

    def __init__(self, context, file_name, flags):
        ContextDerived.__init__(self, context)

        self.file_name = file_name
        self.flags = flags


class JARManifest(ContextDerived):
    """Describes an individual JAR manifest file and how to process it.

    This class isn't very useful for optimizing backends yet because we don't
    capture defines. We can't capture defines safely until all of them are
    defined in moz.build and not Makefile.in files.
    """
    __slots__ = (
        'path',
    )

    def __init__(self, context, path):
        ContextDerived.__init__(self, context)

        self.path = path


class BaseSources(ContextDerived):
    """Base class for files to be compiled during the build."""

    __slots__ = (
        'files',
        'canonical_suffix',
    )

    def __init__(self, context, files, canonical_suffix):
        ContextDerived.__init__(self, context)

        self.files = files
        self.canonical_suffix = canonical_suffix


class Sources(BaseSources):
    """Represents files to be compiled during the build."""

    def __init__(self, context, files, canonical_suffix):
        BaseSources.__init__(self, context, files, canonical_suffix)


class PgoGenerateOnlySources(BaseSources):
    """Represents files to be compiled during the build.

    These files are only used during the PGO generation phase."""

    def __init__(self, context, files):
        BaseSources.__init__(self, context, files, '.cpp')


class GeneratedSources(BaseSources):
    """Represents generated files to be compiled during the build."""

    def __init__(self, context, files, canonical_suffix):
        BaseSources.__init__(self, context, files, canonical_suffix)


class HostSources(HostMixin, BaseSources):
    """Represents files to be compiled for the host during the build."""

    def __init__(self, context, files, canonical_suffix):
        BaseSources.__init__(self, context, files, canonical_suffix)


class HostGeneratedSources(HostMixin, BaseSources):
    """Represents generated files to be compiled for the host during the build."""

    def __init__(self, context, files, canonical_suffix):
        BaseSources.__init__(self, context, files, canonical_suffix)


class UnifiedSources(BaseSources):
    """Represents files to be compiled in a unified fashion during the build."""

    __slots__ = (
        'have_unified_mapping',
        'unified_source_mapping'
    )

    def __init__(self, context, files, canonical_suffix, files_per_unified_file):
        BaseSources.__init__(self, context, files, canonical_suffix)

        self.have_unified_mapping = files_per_unified_file > 1

        if self.have_unified_mapping:
            # Sorted so output is consistent and we don't bump mtimes.
            source_files = list(sorted(self.files))

            # On Windows, path names have a maximum length of 255 characters,
            # so avoid creating extremely long path names.
            unified_prefix = context.relsrcdir
            if len(unified_prefix) > 20:
                unified_prefix = unified_prefix[-20:].split('/', 1)[-1]
            unified_prefix = unified_prefix.replace('/', '_')

            suffix = self.canonical_suffix[1:]
            unified_prefix = 'Unified_%s_%s' % (suffix, unified_prefix)
            self.unified_source_mapping = list(
                group_unified_files(source_files,
                                    unified_prefix=unified_prefix,
                                    unified_suffix=suffix,
                                    files_per_unified_file=files_per_unified_file)
                )


class InstallationTarget(ContextDerived):
    """Describes the rules that affect where files get installed to."""

    __slots__ = (
        'xpiname',
        'subdir',
        'target',
        'enabled'
    )

    def __init__(self, context):
        ContextDerived.__init__(self, context)

        self.xpiname = context.get('XPI_NAME', '')
        self.subdir = context.get('DIST_SUBDIR', '')
        self.target = context['FINAL_TARGET']
        self.enabled = context['DIST_INSTALL'] is not False

    def is_custom(self):
        """Returns whether or not the target is not derived from the default
        given xpiname and subdir."""

        return FinalTargetValue(dict(
            XPI_NAME=self.xpiname,
            DIST_SUBDIR=self.subdir)) == self.target


class FinalTargetFiles(ContextDerived):
    """Sandbox container object for FINAL_TARGET_FILES, which is a
    HierarchicalStringList.

    We need an object derived from ContextDerived for use in the backend, so
    this object fills that role. It just has a reference to the underlying
    HierarchicalStringList, which is created when parsing FINAL_TARGET_FILES.
    """
    __slots__ = ('files')

    def __init__(self, sandbox, files):
        ContextDerived.__init__(self, sandbox)
        self.files = files


class FinalTargetPreprocessedFiles(ContextDerived):
    """Sandbox container object for FINAL_TARGET_PP_FILES, which is a
    HierarchicalStringList.

    We need an object derived from ContextDerived for use in the backend, so
    this object fills that role. It just has a reference to the underlying
    HierarchicalStringList, which is created when parsing
    FINAL_TARGET_PP_FILES.
    """
    __slots__ = ('files')

    def __init__(self, sandbox, files):
        ContextDerived.__init__(self, sandbox)
        self.files = files


class LocalizedFiles(FinalTargetFiles):
    """Sandbox container object for LOCALIZED_FILES, which is a
    HierarchicalStringList.
    """
    pass


class LocalizedPreprocessedFiles(FinalTargetPreprocessedFiles):
    """Sandbox container object for LOCALIZED_PP_FILES, which is a
    HierarchicalStringList.
    """
    pass


class ObjdirFiles(FinalTargetFiles):
    """Sandbox container object for OBJDIR_FILES, which is a
    HierarchicalStringList.
    """
    @property
    def install_target(self):
        return ''


class ObjdirPreprocessedFiles(FinalTargetPreprocessedFiles):
    """Sandbox container object for OBJDIR_PP_FILES, which is a
    HierarchicalStringList.
    """
    @property
    def install_target(self):
        return ''


class TestHarnessFiles(FinalTargetFiles):
    """Sandbox container object for TEST_HARNESS_FILES,
    which is a HierarchicalStringList.
    """
    @property
    def install_target(self):
        return '_tests'


class Exports(FinalTargetFiles):
    """Context derived container object for EXPORTS, which is a
    HierarchicalStringList.

    We need an object derived from ContextDerived for use in the backend, so
    this object fills that role. It just has a reference to the underlying
    HierarchicalStringList, which is created when parsing EXPORTS.
    """
    @property
    def install_target(self):
        return 'dist/include'


class GeneratedFile(ContextDerived):
    """Represents a generated file."""

    __slots__ = (
        'script',
        'method',
        'outputs',
        'inputs',
        'flags',
        'required_before_compile',
        'required_during_compile',
        'localized',
        'force',
    )

    def __init__(self, context, script, method, outputs, inputs,
                 flags=(), localized=False, force=False):
        ContextDerived.__init__(self, context)
        self.script = script
        self.method = method
        self.outputs = outputs if isinstance(outputs, tuple) else (outputs,)
        self.inputs = inputs
        self.flags = flags
        self.localized = localized
        self.force = force

        suffixes = [
            '.h',
            '.inc',
            '.py',
            '.rs',
            # We need to compile Java to generate JNI wrappers for native code
            # compilation to consume.
            'android_apks',
            '.profdata',
            '.webidl'
        ]

        try:
            lib_suffix = context.config.substs['LIB_SUFFIX']
            suffixes.append('.' + lib_suffix)
        except KeyError:
            # Tests may not define LIB_SUFFIX
            pass

        suffixes = tuple(suffixes)

        self.required_before_compile = [
            f for f in self.outputs if f.endswith(suffixes) or 'stl_wrappers/' in f]

        self.required_during_compile = [
            f for f in self.outputs if f.endswith(('.asm', '.c', '.cpp'))]


class ChromeManifestEntry(ContextDerived):
    """Represents a chrome.manifest entry."""

    __slots__ = (
        'path',
        'entry',
    )

    def __init__(self, context, manifest_path, entry):
        ContextDerived.__init__(self, context)
        assert isinstance(entry, ManifestEntry)
        self.path = mozpath.join(self.install_target, manifest_path)
        # Ensure the entry is relative to the directory containing the
        # manifest path.
        entry = entry.rebase(mozpath.dirname(manifest_path))
        # Then add the install_target to the entry base directory.
        self.entry = entry.move(mozpath.dirname(self.path))


class GnProjectData(ContextDerived):
    def __init__(self, context, target_dir, gn_dir_attrs, non_unified_sources):
        ContextDerived.__init__(self, context)
        self.target_dir = target_dir
        self.non_unified_sources = non_unified_sources
        self.gn_input_variables = gn_dir_attrs.variables
        self.gn_sandbox_variables = gn_dir_attrs.sandbox_vars
        self.mozilla_flags = gn_dir_attrs.mozilla_flags
        self.gn_target = gn_dir_attrs.gn_target