author Ryan VanderMeulen <>
Tue, 30 Apr 2013 21:09:41 -0400
changeset 130421 d5db4e09eeef840fd81294ab2a2a0344ea0cde96
parent 130420 c7937fb5f4bbe90cbe8fd030e40d36eeb9f34fb9
child 131410 557f1d26fb712722030373ff369d4dba8477260e
permissions -rw-r--r--
Backed out changesets c7937fb5f4bb and cf8db9b6bd61 (bug 862986) for bustage. CLOSED TREE

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at

from __future__ import unicode_literals

import errno
import logging
import os
import types

from .base import BuildBackend
from import (
from ..util import FileAvoidWrite


DEPTH          := {depth}
topsrcdir      := {topsrc}
srcdir         := {src}
VPATH          := {src}
relativesrcdir := {relsrc}

include {topsrc}/config/

class BackendMakeFile(object):
    """Represents a generated file.

    This is both a wrapper around a file handle as well as a container that
    holds accumulated state.

    It's worth taking a moment to explain the make dependencies. The
    generated as well as the (if it exists) are in the
    GLOBAL_DEPS list. This means that if one of them changes, all targets
    in that Makefile are invalidated. also depends on all of its
    input files.

    It's worth considering the effect of file mtimes on build behavior.

    Since we perform an "all or none" traversal of files (the whole
    tree is scanned as opposed to individual files), if we were to blindly
    write files, the net effect of updating a single mozbuild file
    in the tree is all files have new mtimes. This would in turn
    invalidate all make targets across the whole tree! This would effectively
    undermine incremental builds as any mozbuild change would cause the entire
    tree to rebuild!

    The solution is to not update the mtimes of files unless they
    actually change. We use FileAvoidWrite to accomplish this. However, this
    puts us in a somewhat complicated position when it comes to tree recursion.
    As you are recursing the tree, the first time you come across a
    that is out of date, a full tree build will be incurred. In typical make
    build systems, we would touch the out-of-date target ( to ensure
    its mtime is newer than all its dependencies - even if the contents did
    not change. However, we can't rely on just this approach. During recursion,
    the first trigger of backend generation will cause only that to
    update. If there is another that is also out of date according
    to mtime but whose contents were not changed, when we recurse to that
    directory, make will trigger another full backend generation! This would
    be completely redundant and would slow down builds! This is not acceptable.

    We work around this problem by having backend generation update the mtime
    of if they are older than their inputs - even if the file
    contents did not change. This is essentially a middle ground between
    always updating and only updating the that was out
    of date during recursion.

    def __init__(self, srcdir, objdir, environment):
        self.srcdir = srcdir
        self.objdir = objdir
        self.environment = environment
        self.path = os.path.join(objdir, '')

        # Filenames that influenced the content of this file.
        self.inputs = set()

        self.fh = FileAvoidWrite(self.path)
        self.fh.write('MOZBUILD_DERIVED := 1\n')

        # SUBSTITUTE_FILES handles -> Makefile conversion. This
        # also doubles to handle the case where there is no
        self.fh.write('NO_MAKEFILE_RULE := 1\n')

        # We can't blindly have a SUBMAKEFILES rule because some of the
        # Makefile may not have a corresponding For the case
        # where a new directory is added, the mozbuild file referencing that
        # new directory will need updated. This will cause a full backend
        # scan and build, installing the new Makefile.
        self.fh.write('NO_SUBMAKEFILES_RULE := 1\n')

    def write(self, buf):

    def close(self):
        if self.inputs:
            l = ' '.join(sorted(self.inputs))
            self.fh.write('BACKEND_INPUT_FILES += %s\n' % l)

        result = self.fh.close()

        if not self.inputs:
            return result

        # Update mtime iff any of its input files are newer. See class notes
        # for why we do this.
        existing_mtime = os.path.getmtime(self.path)

        def mtime(path):
                return os.path.getmtime(path)
            except OSError as e:
                if e.errno == errno.ENOENT:
                    return 0


        input_mtime = max(mtime(path) for path in self.inputs)

        if input_mtime > existing_mtime:
            os.utime(self.path, None)

        return result

class RecursiveMakeBackend(BuildBackend):
    """Backend that integrates with the existing recursive make build system.

    This backend facilitates the transition from to

    This backend performs -> Makefile conversion. It also writes
    out .mk files containing content derived from files. Both are
    consumed by the recursive make builder.

    This backend may eventually evolve to write out non-recursive make files.
    However, as long as there are files in the tree, we are tied to
    recursive make and thus will need this backend.

    def _init(self):
        self._backend_files = {}

        self.summary.managed_count = 0
        self.summary.created_count = 0
        self.summary.updated_count = 0
        self.summary.unchanged_count = 0

        def detailed(summary):
            return '{:d} total backend files. {:d} created; {:d} updated; {:d} unchanged'.format(
                summary.managed_count, summary.created_count,
                summary.updated_count, summary.unchanged_count)

        # This is a little kludgy and could be improved with a better API.
        self.summary.backend_detailed_summary = types.MethodType(detailed,

    def _update_from_avoid_write(self, result):
        existed, updated = result

        if not existed:
            self.summary.created_count += 1
        elif updated:
            self.summary.updated_count += 1
            self.summary.unchanged_count += 1

    def consume_object(self, obj):
        """Write out build files necessary to build with recursive make."""

        if not isinstance(obj, SandboxDerived):

        backend_file = self._backend_files.get(obj.srcdir,
            BackendMakeFile(obj.srcdir, obj.objdir, self.get_environment(obj)))

        # Define the paths that will trigger a backend rebuild. We always
        # add because that is proxy for CONFIG. We can't use
        # config.status because there is no make target for that!
        autoconf_path = os.path.join(obj.topobjdir, 'config', '')
        backend_file.inputs |= obj.sandbox_all_paths

        if isinstance(obj, DirectoryTraversal):
            self._process_directory_traversal(obj, backend_file)
        elif isinstance(obj, ConfigFileSubstitution):
            backend_file.write('SUBSTITUTE_FILES += %s\n' % obj.relpath)
            self.summary.managed_count += 1
        elif isinstance(obj, VariablePassthru):
            # Sorted so output is consistent and we don't bump mtimes.
            for k, v in sorted(obj.variables.items()):
                if isinstance(v, list):
                    for item in v:
                        backend_file.write('%s += %s\n' % (k, item))

                    backend_file.write('%s := %s\n' % (k, v))
        elif isinstance(obj, Exports):
            self._process_exports(obj.exports, backend_file)

        elif isinstance(obj, XpcshellManifests):
            self._process_xpcshell_manifests(obj.xpcshell_manifests, backend_file)

        self._backend_files[obj.srcdir] = backend_file

    def consume_finished(self):
        for srcdir in sorted(self._backend_files.keys()):
            bf = self._backend_files[srcdir]

            if not os.path.exists(bf.objdir):
                except OSError as error:
                    if error.errno != errno.EEXIST:

            makefile_in = os.path.join(srcdir, '')
            makefile = os.path.join(bf.objdir, 'Makefile')

            # If exists, use it as a template. Otherwise, create a
            # stub.
            if os.path.exists(makefile_in):
                self.log(logging.DEBUG, 'substitute_makefile',
                    {'path': makefile}, 'Substituting makefile: {path}')

                self.summary.managed_count += 1

                bf.write('SUBSTITUTE_FILES += Makefile\n')
                self.log(logging.DEBUG, 'stub_makefile',
                    {'path': makefile}, 'Creating stub Makefile: {path}')

                params = {
                    'topsrc': bf.environment.get_top_srcdir(makefile),
                    'src': bf.environment.get_file_srcdir(makefile),
                    'depth': bf.environment.get_depth(makefile),
                    'relsrc': bf.environment.get_relative_srcdir(makefile),

                aw = FileAvoidWrite(makefile)
                self.summary.managed_count += 1

            self.summary.managed_count += 1

    def _process_directory_traversal(self, obj, backend_file):
        """Process a data.DirectoryTraversal instance."""
        fh = backend_file.fh

        for tier, dirs in obj.tier_dirs.iteritems():
            fh.write('TIERS += %s\n' % tier)

            if dirs:
                fh.write('tier_%s_dirs += %s\n' % (tier, ' '.join(dirs)))

            # tier_static_dirs should have the same keys as tier_dirs.
            if obj.tier_static_dirs[tier]:
                fh.write('tier_%s_staticdirs += %s\n' % (
                    tier, ' '.join(obj.tier_static_dirs[tier])))

                static = ' '.join(obj.tier_static_dirs[tier])
                fh.write('EXTERNAL_DIRS += %s\n' % static)

        if obj.dirs:
            fh.write('DIRS := %s\n' % ' '.join(obj.dirs))

        if obj.parallel_dirs:
            fh.write('PARALLEL_DIRS := %s\n' % ' '.join(obj.parallel_dirs))

        if obj.tool_dirs:
            fh.write('TOOL_DIRS := %s\n' % ' '.join(obj.tool_dirs))

        if obj.test_dirs:
            fh.write('TEST_DIRS := %s\n' % ' '.join(obj.test_dirs))

        if obj.test_tool_dirs and \
            self.environment.substs.get('ENABLE_TESTS', False):

            fh.write('TOOL_DIRS += %s\n' % ' '.join(obj.test_tool_dirs))

        if len(obj.external_make_dirs):
            fh.write('DIRS += %s\n' % ' '.join(obj.external_make_dirs))

        if len(obj.parallel_external_make_dirs):
            fh.write('PARALLEL_DIRS += %s\n' %
                ' '.join(obj.parallel_external_make_dirs))

    def _process_exports(self, exports, backend_file, namespace=""):
        strings = exports.get_strings()
        if namespace:
            if strings:
                backend_file.write('EXPORTS_NAMESPACES += %s\n' % namespace)
            export_name = 'EXPORTS_%s' % namespace
            namespace += '/'
            export_name = 'EXPORTS'

        # Iterate over the list of export filenames, printing out an EXPORTS
        # declaration for each.
        if strings:
            backend_file.write('%s += %s\n' % (export_name, ' '.join(strings)))

        children = exports.get_children()
        for subdir in sorted(children):
            self._process_exports(children[subdir], backend_file,
                                  namespace=namespace + subdir)

    def _process_xpcshell_manifests(self, manifest, backend_file, namespace=""):
        backend_file.write('XPCSHELL_TESTS += %s\n' % os.path.dirname(manifest))