scripts/sync-hg-repos-from-manifest
author Dan Minor <dminor@mozilla.com>
Fri, 18 Dec 2015 15:12:07 -0500
changeset 362083 2248273cff1c9789e941b14cb0a303dbfca0ff98
parent 361842 3400564e52a6f1a8601efdea0d12c00d2da6849a
child 362435 3ec5e599f6492008fbce86e5f4a525914527b2cc
permissions -rwxr-xr-x
mozreview: people besides review requesters should be able autoland (bug 1205018) r=mdoglio This removes the check for whether or not the review is mutable by the current user, which allows people other than the review requester to autoland provided they have sufficient privileges.

#!/usr/bin/env python2.7
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""Synchronize Mercurial repositories from a manifest.

Given a manifest of known repositories, ensure local clones of those
repositories are up to date.
"""

import argparse
import errno
import grp
import multiprocessing
import os
import pwd
import subprocess

import concurrent.futures as futures


HERE = os.path.abspath(os.path.dirname(__file__))
repo_permissions = os.path.join(HERE, 'repo-permissions')
if not os.path.exists(repo_permissions):
    raise Exception('could not find repo-permissions script')


# We need to use the hg that is configured for replication otherwise
# hooks and other Python dependencies may not load properly.
HG = '/repo/hg/venv_replication/bin/hg'
if not os.path.exists(HG):
    raise Exception('could not find %s' % HG)


class CloneError(Exception):
    def __init__(self, path, is_clone, output=None):
        super(CloneError, self).__init__()

        self.path = path
        self.is_clone = is_clone
        self.output = output


def clone_or_pull(rel, path, requires, url, user, group):
    is_clone = False

    try:
        if not os.path.exists(path):
            is_clone = True
            args = [HG, 'init', path]
            if 'generaldelta' in requires:
                args.extend(['--config', 'format.generaldelta=true'])

            print('creating empty repo: %s' % path)
            subprocess.check_output(args, cwd='/', stderr=subprocess.STDOUT)

        args = [HG, '-R', path, 'pull', url]
        subprocess.check_output(args, cwd='/', stderr=subprocess.STDOUT)

        # Synchronize hgrc file via mirror-pull.
        # This should ideally be part of `hg pull`.
        args = ['/usr/local/bin/mirror-pull', '--hgrc', rel]
        subprocess.check_output(args, cwd='/', stderr=subprocess.STDOUT)

        # Adjust repository permissions.
        args = [repo_permissions, path, user, group, 'wwr']
        subprocess.check_output(args, cwd='/', stderr=subprocess.STDOUT)

    except subprocess.CalledProcessError as e:
        raise CloneError(path, is_clone, output=e.output)
    except Exception as e:
        raise CloneError(path, is_clone, e.message)

    return rel, is_clone

def synchronize_repos(repos, dest_dir, base_url, user, group):
    # Do an initial pass to create the directory scaffolding.
    dirs = set()
    for rel, requires in repos:
        full = os.path.join(dest_dir, rel)
        dirs.add(os.path.dirname(full))

    for d in dirs:
        try:
            os.makedirs(d, mode=0775)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

    with futures.ThreadPoolExecutor(multiprocessing.cpu_count()) as e:
        fs = []
        for rel, requires in repos:
            full = os.path.join(dest_dir, rel)
            url = '%s/%s' % (base_url.rstrip('/'), rel)
            fs.append(e.submit(clone_or_pull, rel, full, requires, url,
                               user, group))

        for f in futures.as_completed(fs):
            if f.exception() is not None:
                e = f.exception()
                print('ERROR syncing %s: %s' % (e.path, e.output))
                continue

            path, is_clone = f.result()
            if is_clone:
                print('%s cloned successfully' % path)
            else:
                print('%s pulled successfully' % path)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('manifest',
                        help='File containing list of repos to sync')
    parser.add_argument('dest_dir',
                        help='Directory where repositories should be created')
    parser.add_argument('base_url',
                        help='Base URL from which to clone repositories')

    args = parser.parse_args()

    repos = []
    with open(args.manifest, 'rb') as fh:
        for line in fh:
            line = line.rstrip('\n')
            rel, requires = line.split('\t')
            requires = set(requires.split(','))
            repos.append((rel, requires))

    user = pwd.getpwuid(os.getuid())
    group = grp.getgrgid(os.getgid())

    synchronize_repos(repos, args.dest_dir, args.base_url, user.pw_name,
                      group.gr_name)