taskcluster/taskgraph/files_changed.py
author Greyson Gilbert <greyson.gilbert.oss@gmail.com>
Tue, 30 Jul 2019 23:32:29 +0000
changeset 549189 6019fc32fbf36e44a00248aa12f94c6f111a9213
parent 507777 4e1a901cc0a5b0ca50f00cc929a967e0e6c0063e
permissions -rw-r--r--
Bug 1564293 - Allow efficient slicewise upload of 3D textures r=jgilbert Before this patch any partial upload to a texture would incur a zeroing of the texture first to prevent leakage of information. The texture now tracks, for each image not fully initialized, which z-slices have been initialized, and only zeroes the rest of the slices when the texture is used. Differential Revision: https://phabricator.services.mozilla.com/D37843

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""
Support for optimizing tasks based on the set of files that have changed.
"""

from __future__ import absolute_import, print_function, unicode_literals

import logging
import requests
from redo import retry
from mozpack.path import match as mozpackmatch, join as join_path
from mozversioncontrol import get_repository_object, InvalidRepoPath
from subprocess import CalledProcessError
from mozbuild.util import memoize

logger = logging.getLogger(__name__)


@memoize
def get_changed_files(repository, revision):
    """
    Get the set of files changed in the push headed by the given revision.
    Responses are cached, so multiple calls with the same arguments are OK.
    """
    url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision)
    logger.debug("Querying version control for metadata: %s", url)

    def get_automationrelevance():
        response = requests.get(url, timeout=30)
        return response.json()
    contents = retry(get_automationrelevance, attempts=10, sleeptime=10)

    logger.debug('{} commits influencing task scheduling:'
                 .format(len(contents['changesets'])))
    changed_files = set()
    for c in contents['changesets']:
        desc = ""  # Support empty desc
        if c['desc']:
            desc = c['desc'].splitlines()[0].encode('ascii', 'ignore')
        logger.debug(" {cset} {desc}".format(
            cset=c['node'][0:12],
            desc=desc))
        changed_files |= set(c['files'])

    return changed_files


def check(params, file_patterns):
    """Determine whether any of the files changed in the indicated push to
    https://hg.mozilla.org match any of the given file patterns."""
    repository = params.get('head_repository')
    revision = params.get('head_rev')
    if not repository or not revision:
        logger.warning("Missing `head_repository` or `head_rev` parameters; "
                       "assuming all files have changed")
        return True

    changed_files = get_changed_files(repository, revision)

    if 'comm_head_repository' in params:
        repository = params.get('comm_head_repository')
        revision = params.get('comm_head_rev')
        if not revision:
            logger.warning("Missing `comm_head_rev` parameters; "
                           "assuming all files have changed")
            return True

        changed_files |= {
            join_path("comm", file) for file in
            get_changed_files(repository, revision)
        }

    for pattern in file_patterns:
        for path in changed_files:
            if mozpackmatch(path, pattern):
                return True

    return False


@memoize
def get_locally_changed_files(repo):
    try:
        vcs = get_repository_object(repo)
        return set(vcs.get_outgoing_files('AM'))
    except (InvalidRepoPath, CalledProcessError):
        return set()