bug 763903: regularly run mozconfig comparisons for firefox. r=ted/bhearsum
authorBen Hearsum <bhearsum@mozilla.com>
Mon, 15 Jul 2013 13:27:12 -0400
changeset 150821 d8f26f00c82f12df862edd0a36d7f47b5848362a
parent 150820 dc6fc156140e4197789d938d68b3a13a7f2535e4
child 150822 bf847b1a3776f8f057a6815ae8639216805cebd8
push id2859
push userakeybl@mozilla.com
push dateMon, 16 Sep 2013 19:14:59 +0000
treeherdermozilla-beta@87d3c51cd2bf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted, bhearsum
bugs763903
milestone25.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 763903: regularly run mozconfig comparisons for firefox. r=ted/bhearsum
browser/Makefile.in
browser/config/mozconfigs/whitelist
build/compare-mozconfig/compare-mozconfigs-wrapper.py
build/compare-mozconfig/compare-mozconfigs.py
build/release/info.py
build/release/sanity.py
build/util/hg.py
--- a/browser/Makefile.in
+++ b/browser/Makefile.in
@@ -14,8 +14,11 @@ ifdef MAKENSISU
 # For Windows build the uninstaller during the application build since the
 # uninstaller is included with the application for mar file generation.
 libs::
 	$(MAKE) -C installer/windows uninstaller
 ifdef MOZ_MAINTENANCE_SERVICE
 	$(MAKE) -C installer/windows maintenanceservice_installer
 endif
 endif
+
+check::
+	$(PYTHON) $(topsrcdir)/build/compare-mozconfig/compare-mozconfigs-wrapper.py
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/whitelist
@@ -0,0 +1,90 @@
+# 'nightly' contains things that are in nightly mozconfigs and allowed to be missing from release builds.
+# Other keys in whitelist contain things are in that branches mozconfigs and allowed to be missing from nightly builds.
+whitelist = {
+    'release': {},
+    'nightly': {},
+    }
+
+all_platforms = ['win32', 'linux32', 'linux64', 'macosx-universal']
+
+for platform in all_platforms:
+    whitelist['nightly'][platform] = [
+        'ac_add_options --enable-update-channel=nightly',
+        'ac_add_options --enable-profiling',
+        'mk_add_options CLIENT_PY_ARGS="--hg-options=\'--verbose --time\' --hgtool=../tools/buildfarm/utils/hgtool.py --skip-chatzilla --skip-comm --skip-inspector --skip-venkman --tinderbox-print"'
+    ]
+
+for platform in ['linux32', 'linux64', 'macosx-universal']:
+    whitelist['nightly'][platform] += [
+        'ac_add_options --enable-codesighs',
+        'mk_add_options MOZ_MAKE_FLAGS="-j4"',
+    ]
+
+for platform in ['linux32', 'linux64', 'macosx-universal', 'win32']:
+    whitelist['nightly'][platform] += ['ac_add_options --enable-signmar']
+    whitelist['nightly'][platform] += ['ac_add_options --enable-js-diagnostics']
+
+whitelist['nightly']['linux32'] += [
+    'CXX=$REAL_CXX',
+    'CXX="ccache $REAL_CXX"',
+    'CC="ccache $REAL_CC"',
+    'mk_add_options PROFILE_GEN_SCRIPT=@TOPSRCDIR@/build/profile_pageloader.pl',
+    'ac_add_options --with-ccache=/usr/bin/ccache',
+    'export MOZILLA_OFFICIAL=1',
+    'export MOZ_TELEMETRY_REPORTING=1',
+    "mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'",
+    'STRIP_FLAGS="--strip-debug"',
+    'ac_add_options --disable-elf-hack # --enable-elf-hack conflicts with --enable-profiling',
+]
+
+whitelist['nightly']['linux64'] += [
+    'export MOZILLA_OFFICIAL=1',
+    'export MOZ_TELEMETRY_REPORTING=1',
+    "mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'",
+    'STRIP_FLAGS="--strip-debug"',
+    'ac_add_options --with-ccache=/usr/bin/ccache',
+    'ac_add_options --disable-elf-hack # --enable-elf-hack conflicts with --enable-profiling',
+]
+
+whitelist['nightly']['macosx-universal'] += [
+    'ac_add_options --with-macbundlename-prefix=Firefox',
+    'mk_add_options MOZ_MAKE_FLAGS="-j12"',
+    'ac_add_options --with-ccache',
+    'ac_add_options --disable-install-strip',
+    'ac_add_options --enable-instruments',
+    'ac_add_options --enable-dtrace',
+]
+
+whitelist['nightly']['win32'] += [
+    '. $topsrcdir/configs/mozilla2/win32/include/choose-make-flags',
+    'mk_add_options MOZ_MAKE_FLAGS=-j1',
+    'if test "$IS_NIGHTLY" != ""; then',
+    'ac_add_options --disable-auto-deps',
+    'fi',
+    'ac_add_options --enable-metro',
+]
+
+for platform in all_platforms:
+    whitelist['release'][platform] = [
+        'ac_add_options --enable-update-channel=release',
+        'ac_add_options --enable-official-branding',
+        'mk_add_options MOZ_MAKE_FLAGS="-j4"',
+        'export BUILDING_RELEASE=1',
+    ]
+whitelist['release']['win32'] += ['mk_add_options MOZ_PGO=1']
+whitelist['release']['linux32'] += [
+    'export MOZILLA_OFFICIAL=1',
+    'export MOZ_TELEMETRY_REPORTING=1',
+    'mk_add_options MOZ_PGO=1',
+    "mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'",
+]
+whitelist['release']['linux64'] += [
+    'export MOZILLA_OFFICIAL=1',
+    'export MOZ_TELEMETRY_REPORTING=1',
+    'mk_add_options MOZ_PGO=1',
+    "mk_add_options PROFILE_GEN_SCRIPT='$(PYTHON) @MOZ_OBJDIR@/_profile/pgo/profileserver.py 10'",
+]
+
+if __name__ == '__main__':
+    import pprint
+    pprint.pprint(whitelist)
new file mode 100644
--- /dev/null
+++ b/build/compare-mozconfig/compare-mozconfigs-wrapper.py
@@ -0,0 +1,53 @@
+#!/usr/bin/python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import subprocess
+import sys
+from os import path
+from buildconfig import substs
+
+def determine_platform():
+    platform_mapping = {'WINNT': {'x86_64': 'win64',
+                                  'i386': 'win32'},
+                        'Darwin': {'x86_64': 'macosx-universal',
+                                   'i386':'macosx-universal'},
+                        'Linux': {'x86_64': 'linux64',
+                                  'i386': 'linux32'}}
+
+    os_type = substs['OS_TARGET']
+    cpu_type = substs['TARGET_CPU']
+    return platform_mapping.get(os_type, {}).get(cpu_type, None)
+
+def main():
+    """ A wrapper script that calls compare-mozconfig.py
+    based on the platform that the machine is building for"""
+    platform = determine_platform()
+
+    if platform is not None:
+        python_exe = substs['PYTHON']
+        topsrcdir = substs['top_srcdir']
+
+        # construct paths and args for compare-mozconfig
+        browser_dir = path.join(topsrcdir, 'browser')
+        script_path = path.join(topsrcdir, 'build/compare-mozconfig/compare-mozconfigs.py')
+        whitelist_path = path.join(browser_dir, 'config/mozconfigs/whitelist')
+        beta_mozconfig_path = path.join(browser_dir, 'config/mozconfigs', platform, 'beta')
+        release_mozconfig_path = path.join(browser_dir, 'config/mozconfigs', platform, 'release')
+        nightly_mozconfig_path = path.join(browser_dir, 'config/mozconfigs', platform, 'nightly')
+
+        # compare beta vs nightly
+        ret_code = subprocess.call([python_exe, script_path, '--whitelist',
+                                    whitelist_path, '--no-download',
+                                    platform + ',' + beta_mozconfig_path +
+                                    ',' + nightly_mozconfig_path])
+
+        if ret_code > 0:
+            sys.exit(ret_code)
+
+        # compare release vs nightly
+        ret_code = subprocess.call([python_exe, script_path, '--whitelist',
+                                    whitelist_path, '--no-download',
+                                    platform + ',' + release_mozconfig_path +
+                                    ',' + nightly_mozconfig_path])
new file mode 100644
--- /dev/null
+++ b/build/compare-mozconfig/compare-mozconfigs.py
@@ -0,0 +1,68 @@
+#!/usr/bin/python
+import logging
+import os
+import site
+import sys
+import urllib2
+
+site.addsitedir(os.path.join(os.path.dirname(__file__), "../../lib/python"))
+
+from release.sanity import verify_mozconfigs
+from release.info import readConfig
+from util.hg import make_hg_url
+
+FAILURE_CODE = 1
+SUCCESS_CODE = 0
+
+def get_mozconfig(path, options):
+    """Consumes a path and returns a list of lines from
+    the mozconfig file. If download is required, the path
+    specified should be relative to the root of the hg
+    repository e.g browser/config/mozconfigs/linux32/nightly"""
+    if options.no_download:
+        return open(path, 'r').readlines()
+    else:
+        url = make_hg_url(options.hghost, options.branch, 'http',
+                    options.revision, path)
+        return urllib2.urlopen(url).readlines()
+
+if __name__ == '__main__':
+    from optparse import OptionParser
+    parser = OptionParser()
+
+    parser.add_option('--branch', dest='branch')
+    parser.add_option('--revision', dest='revision')
+    parser.add_option('--hghost', dest='hghost', default='hg.mozilla.org')
+    parser.add_option('--whitelist', dest='whitelist')
+    parser.add_option('--no-download', action='store_true', dest='no_download',
+                      default=False)
+    options, args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO)
+
+    missing_args = options.branch is None or options.revision is None
+    if not options.no_download and missing_args:
+        logging.error('Not enough arguments to download mozconfigs')
+        sys.exit(FAILURE_CODE)
+
+    mozconfig_whitelist = readConfig(options.whitelist, ['whitelist'])
+
+    for arg in args:
+        platform, mozconfig_path, nightly_mozconfig_path = arg.split(',')
+
+        mozconfig_lines = get_mozconfig(mozconfig_path, options)
+        nightly_mozconfig_lines = get_mozconfig(nightly_mozconfig_path, options)
+
+        mozconfig_pair = (mozconfig_path, mozconfig_lines)
+        nightly_mozconfig_pair = (nightly_mozconfig_path,
+                                  nightly_mozconfig_lines)
+
+        passed = verify_mozconfigs(mozconfig_pair, nightly_mozconfig_pair,
+                                   platform, mozconfig_whitelist)
+
+        if passed:
+            logging.info('Mozconfig check passed!')
+        else:
+            logging.error('Mozconfig check failed!')
+            sys.exit(FAILURE_CODE)
+    sys.exit(SUCCESS_CODE)
new file mode 100644
--- /dev/null
+++ b/build/release/info.py
@@ -0,0 +1,218 @@
+from datetime import datetime
+import os
+from os import path
+import re
+import shutil
+import sys
+from urllib2 import urlopen
+
+from release.paths import makeCandidatesDir
+
+import logging
+log = logging.getLogger(__name__)
+
+# If version has two parts with no trailing specifiers like "rc", we
+# consider it a "final" release for which we only create a _RELEASE tag.
+FINAL_RELEASE_REGEX = "^\d+\.\d+$"
+
+
+class ConfigError(Exception):
+    pass
+
+
+def getBuildID(platform, product, version, buildNumber, nightlyDir='nightly',
+               server='stage.mozilla.org'):
+    infoTxt = makeCandidatesDir(product, version, buildNumber, nightlyDir,
+                                protocol='http', server=server) + \
+        '%s_info.txt' % platform
+    try:
+        buildInfo = urlopen(infoTxt).read()
+    except:
+        log.error("Failed to retrieve %s" % infoTxt)
+        raise
+
+    for line in buildInfo.splitlines():
+        key, value = line.rstrip().split('=', 1)
+        if key == 'buildID':
+            return value
+
+
+def findOldBuildIDs(product, version, buildNumber, platforms,
+                    nightlyDir='nightly', server='stage.mozilla.org'):
+    ids = {}
+    if buildNumber <= 1:
+        return ids
+    for n in range(1, buildNumber):
+        for platform in platforms:
+            if platform not in ids:
+                ids[platform] = []
+            try:
+                id = getBuildID(platform, product, version, n, nightlyDir,
+                                server)
+                ids[platform].append(id)
+            except Exception, e:
+                log.error("Hit exception: %s" % e)
+    return ids
+
+
+def getReleaseConfigName(product, branch, version=None, staging=False):
+    # XXX: Horrible hack for bug 842741. Because Thunderbird release
+    # and esr both build out of esr17 repositories we'll bump the wrong
+    # config for release without this.
+    if product == 'thunderbird' and 'esr17' in branch and version and 'esr' not in version:
+        cfg = 'release-thunderbird-comm-release.py'
+    else:
+        cfg = 'release-%s-%s.py' % (product, branch)
+    if staging:
+        cfg = 'staging_%s' % cfg
+    return cfg
+
+
+def readReleaseConfig(configfile, required=[]):
+    return readConfig(configfile, keys=['releaseConfig'], required=required)
+
+
+def readBranchConfig(dir, localconfig, branch, required=[]):
+    shutil.copy(localconfig, path.join(dir, "localconfig.py"))
+    oldcwd = os.getcwd()
+    os.chdir(dir)
+    sys.path.append(".")
+    try:
+        return readConfig("config.py", keys=['BRANCHES', branch],
+                          required=required)
+    finally:
+        os.chdir(oldcwd)
+        sys.path.remove(".")
+
+
+def readConfig(configfile, keys=[], required=[]):
+    c = {}
+    execfile(configfile, c)
+    for k in keys:
+        c = c[k]
+    items = c.keys()
+    err = False
+    for key in required:
+        if key not in items:
+            err = True
+            log.error("Required item `%s' missing from %s" % (key, c))
+    if err:
+        raise ConfigError("Missing at least one item in config, see above")
+    return c
+
+
+def isFinalRelease(version):
+    return bool(re.match(FINAL_RELEASE_REGEX, version))
+
+
+def getBaseTag(product, version):
+    product = product.upper()
+    version = version.replace('.', '_')
+    return '%s_%s' % (product, version)
+
+
+def getTags(baseTag, buildNumber, buildTag=True):
+    t = ['%s_RELEASE' % baseTag]
+    if buildTag:
+        t.append('%s_BUILD%d' % (baseTag, int(buildNumber)))
+    return t
+
+
+def getRuntimeTag(tag):
+    return "%s_RUNTIME" % tag
+
+
+def getReleaseTag(tag):
+    return "%s_RELEASE" % tag
+
+
+def generateRelbranchName(version, prefix='GECKO'):
+    return '%s%s_%s_RELBRANCH' % (
+        prefix, version.replace('.', ''),
+        datetime.now().strftime('%Y%m%d%H'))
+
+
+def getReleaseName(product, version, buildNumber):
+    return '%s-%s-build%s' % (product.title(), version, str(buildNumber))
+
+
+def getRepoMatchingBranch(branch, sourceRepositories):
+    for sr in sourceRepositories.values():
+        if branch in sr['path']:
+            return sr
+    return None
+
+
+def fileInfo(filepath, product):
+    """Extract information about a release file.  Returns a dictionary with the
+    following keys set:
+    'product', 'version', 'locale', 'platform', 'contents', 'format',
+    'pathstyle'
+
+    'contents' is one of 'complete', 'installer'
+    'format' is one of 'mar' or 'exe'
+    'pathstyle' is either 'short' or 'long', and refers to if files are all in
+        one directory, with the locale as part of the filename ('short' paths,
+        firefox 3.0 style filenames), or if the locale names are part of the
+        directory structure, but not the file name itself ('long' paths,
+        firefox 3.5+ style filenames)
+    """
+    try:
+        # Mozilla 1.9.0 style (aka 'short') paths
+        # e.g. firefox-3.0.12.en-US.win32.complete.mar
+        filename = os.path.basename(filepath)
+        m = re.match("^(%s)-([0-9.]+)\.([-a-zA-Z]+)\.(win32)\.(complete|installer)\.(mar|exe)$" % product, filename)
+        if not m:
+            raise ValueError("Could not parse: %s" % filename)
+        return {'product': m.group(1),
+                'version': m.group(2),
+                'locale': m.group(3),
+                'platform': m.group(4),
+                'contents': m.group(5),
+                'format': m.group(6),
+                'pathstyle': 'short',
+                'leading_path': '',
+                }
+    except:
+        # Mozilla 1.9.1 and on style (aka 'long') paths
+        # e.g. update/win32/en-US/firefox-3.5.1.complete.mar
+        #      win32/en-US/Firefox Setup 3.5.1.exe
+        ret = {'pathstyle': 'long'}
+        if filepath.endswith('.mar'):
+            ret['format'] = 'mar'
+            m = re.search("update/(win32|linux-i686|linux-x86_64|mac|mac64)/([-a-zA-Z]+)/(%s)-(\d+\.\d+(?:\.\d+)?(?:\w+(?:\d+)?)?)\.(complete)\.mar" % product, filepath)
+            if not m:
+                raise ValueError("Could not parse: %s" % filepath)
+            ret['platform'] = m.group(1)
+            ret['locale'] = m.group(2)
+            ret['product'] = m.group(3)
+            ret['version'] = m.group(4)
+            ret['contents'] = m.group(5)
+            ret['leading_path'] = ''
+        elif filepath.endswith('.exe'):
+            ret['format'] = 'exe'
+            ret['contents'] = 'installer'
+            # EUballot builds use a different enough style of path than others
+            # that we can't catch them in the same regexp
+            if filepath.find('win32-EUballot') != -1:
+                ret['platform'] = 'win32'
+                m = re.search("(win32-EUballot/)([-a-zA-Z]+)/((?i)%s) Setup (\d+\.\d+(?:\.\d+)?(?:\w+\d+)?(?:\ \w+\ \d+)?)\.exe" % product, filepath)
+                if not m:
+                    raise ValueError("Could not parse: %s" % filepath)
+                ret['leading_path'] = m.group(1)
+                ret['locale'] = m.group(2)
+                ret['product'] = m.group(3).lower()
+                ret['version'] = m.group(4)
+            else:
+                m = re.search("(partner-repacks/[-a-zA-Z0-9_]+/|)(win32|mac|linux-i686)/([-a-zA-Z]+)/((?i)%s) Setup (\d+\.\d+(?:\.\d+)?(?:\w+(?:\d+)?)?(?:\ \w+\ \d+)?)\.exe" % product, filepath)
+                if not m:
+                    raise ValueError("Could not parse: %s" % filepath)
+                ret['leading_path'] = m.group(1)
+                ret['platform'] = m.group(2)
+                ret['locale'] = m.group(3)
+                ret['product'] = m.group(4).lower()
+                ret['version'] = m.group(5)
+        else:
+            raise ValueError("Unknown filetype for %s" % filepath)
+
+        return ret
new file mode 100644
--- /dev/null
+++ b/build/release/sanity.py
@@ -0,0 +1,124 @@
+import difflib
+import logging
+import re
+import urllib2
+from util.commands import run_cmd, get_output
+from util.hg import get_repo_name, make_hg_url
+from subprocess import CalledProcessError
+
+log = logging.getLogger(__name__)
+
+
+def check_buildbot():
+    """check if buildbot command works"""
+    try:
+        run_cmd(['buildbot', '--version'])
+    except CalledProcessError:
+        log.error("FAIL: buildbot command doesn't work", exc_info=True)
+        raise
+
+
+def find_version(contents, versionNumber):
+    """Given an open readable file-handle look for the occurrence
+       of the version # in the file"""
+    ret = re.search(re.compile(re.escape(versionNumber), re.DOTALL), contents)
+    return ret
+
+
+def locale_diff(locales1, locales2):
+    """ accepts two lists and diffs them both ways, returns any differences
+    found """
+    diff_list = [locale for locale in locales1 if not locale in locales2]
+    diff_list.extend(locale for locale in locales2 if not locale in locales1)
+    return diff_list
+
+
+def get_buildbot_username_param():
+    cmd = ['buildbot', 'sendchange', '--help']
+    output = get_output(cmd)
+    if "-W, --who=" in output:
+        return "--who"
+    else:
+        return "--username"
+
+
+def sendchange(branch, revision, username, master, products):
+    """Send the change to buildbot to kick off the release automation"""
+    if isinstance(products, basestring):
+        products = [products]
+    cmd = [
+        'buildbot',
+        'sendchange',
+        get_buildbot_username_param(),
+        username,
+        '--master',
+        master,
+        '--branch',
+        branch,
+        '-p',
+        'products:%s' % ','.join(products),
+        '-p',
+        'script_repo_revision:%s' % revision,
+        'release_build'
+    ]
+    logging.info("Executing: %s" % cmd)
+    run_cmd(cmd)
+
+
+def verify_mozconfigs(mozconfig_pair, nightly_mozconfig_pair, platform,
+                      mozconfigWhitelist={}):
+    """Compares mozconfig to nightly_mozconfig and compare to an optional
+    whitelist of known differences. mozconfig_pair and nightly_mozconfig_pair
+    are pairs containing the mozconfig's identifier and the list of lines in
+    the mozconfig."""
+
+    # unpack the pairs to get the names, the names are just for
+    # identifying the mozconfigs when logging the error messages
+    mozconfig_name, mozconfig_lines = mozconfig_pair
+    nightly_mozconfig_name, nightly_mozconfig_lines = nightly_mozconfig_pair
+
+    missing_args = mozconfig_lines == [] or nightly_mozconfig_lines == []
+    if missing_args:
+        log.info("Missing mozconfigs to compare for %s" % platform)
+        return False
+
+    success = True
+
+    diffInstance = difflib.Differ()
+    diff_result = diffInstance.compare(mozconfig_lines, nightly_mozconfig_lines)
+    diffList = list(diff_result)
+
+    for line in diffList:
+        clean_line = line[1:].strip()
+        if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
+            # skip comment lines
+            if clean_line.startswith('#'):
+                continue
+            # compare to whitelist
+            message = ""
+            if line[0] == '-':
+                if platform in mozconfigWhitelist.get('release', {}):
+                    if clean_line in \
+                            mozconfigWhitelist['release'][platform]:
+                        continue
+            elif line[0] == '+':
+                if platform in mozconfigWhitelist.get('nightly', {}):
+                    if clean_line in \
+                            mozconfigWhitelist['nightly'][platform]:
+                        continue
+                    else:
+                        log.warning("%s not in %s %s!" % (
+                            clean_line, platform,
+                            mozconfigWhitelist['nightly'][platform]))
+            else:
+                log.error("Skipping line %s!" % line)
+                continue
+            message = "found in %s but not in %s: %s"
+            if line[0] == '-':
+                log.error(message % (mozconfig_name,
+                                     nightly_mozconfig_name, clean_line))
+            else:
+                log.error(message % (nightly_mozconfig_name,
+                                     mozconfig_name, clean_line))
+            success = False
+    return success
new file mode 100644
--- /dev/null
+++ b/build/util/hg.py
@@ -0,0 +1,611 @@
+"""Functions for interacting with hg"""
+import os
+import re
+import subprocess
+from urlparse import urlsplit
+from ConfigParser import RawConfigParser
+
+from util.commands import run_cmd, get_output, remove_path
+from util.retry import retry
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class DefaultShareBase:
+    pass
+DefaultShareBase = DefaultShareBase()
+
+
+class HgUtilError(Exception):
+    pass
+
+
+def _make_absolute(repo):
+    if repo.startswith("file://"):
+        path = repo[len("file://"):]
+        repo = "file://%s" % os.path.abspath(path)
+    elif "://" not in repo:
+        repo = os.path.abspath(repo)
+    return repo
+
+
+def make_hg_url(hgHost, repoPath, protocol='https', revision=None,
+                filename=None):
+    """construct a valid hg url from a base hg url (hg.mozilla.org),
+    repoPath, revision and possible filename"""
+    base = '%s://%s' % (protocol, hgHost)
+    repo = '/'.join(p.strip('/') for p in [base, repoPath])
+    if not filename:
+        if not revision:
+            return repo
+        else:
+            return '/'.join([p.strip('/') for p in [repo, 'rev', revision]])
+    else:
+        assert revision
+        return '/'.join([p.strip('/') for p in [repo, 'raw-file', revision, filename]])
+
+
+def get_repo_name(repo):
+    return repo.rstrip('/').split('/')[-1]
+
+
+def get_repo_path(repo):
+    repo = _make_absolute(repo)
+    if repo.startswith("/"):
+        return repo.lstrip("/")
+    else:
+        return urlsplit(repo).path.lstrip("/")
+
+
+def get_revision(path):
+    """Returns which revision directory `path` currently has checked out."""
+    return get_output(['hg', 'parent', '--template', '{node|short}'], cwd=path)
+
+
+def get_branch(path):
+    return get_output(['hg', 'branch'], cwd=path).strip()
+
+
+def get_branches(path):
+    branches = []
+    for line in get_output(['hg', 'branches', '-c'], cwd=path).splitlines():
+        branches.append(line.split()[0])
+    return branches
+
+
+def is_hg_cset(rev):
+    """Retruns True if passed revision represents a valid HG revision
+    (long or short(er) 40 bit hex)"""
+    try:
+        int(rev, 16)
+        return True
+    except (TypeError, ValueError):
+        return False
+
+
+def hg_ver():
+    """Returns the current version of hg, as a tuple of
+    (major, minor, build)"""
+    ver_string = get_output(['hg', '-q', 'version'])
+    match = re.search("\(version ([0-9.]+)\)", ver_string)
+    if match:
+        bits = match.group(1).split(".")
+        if len(bits) < 3:
+            bits += (0,)
+        ver = tuple(int(b) for b in bits)
+    else:
+        ver = (0, 0, 0)
+    log.debug("Running hg version %s", ver)
+    return ver
+
+
+def purge(dest):
+    """Purge the repository of all untracked and ignored files."""
+    try:
+        run_cmd(['hg', '--config', 'extensions.purge=', 'purge',
+                 '-a', '--all', dest], cwd=dest)
+    except subprocess.CalledProcessError, e:
+        log.debug('purge failed: %s' % e)
+        raise
+
+
+def update(dest, branch=None, revision=None):
+    """Updates working copy `dest` to `branch` or `revision`.  If neither is
+    set then the working copy will be updated to the latest revision on the
+    current branch.  Local changes will be discarded."""
+    # If we have a revision, switch to that
+    if revision is not None:
+        cmd = ['hg', 'update', '-C', '-r', revision]
+        run_cmd(cmd, cwd=dest)
+    else:
+        # Check & switch branch
+        local_branch = get_output(['hg', 'branch'], cwd=dest).strip()
+
+        cmd = ['hg', 'update', '-C']
+
+        # If this is different, checkout the other branch
+        if branch and branch != local_branch:
+            cmd.append(branch)
+
+        run_cmd(cmd, cwd=dest)
+    return get_revision(dest)
+
+
+def clone(repo, dest, branch=None, revision=None, update_dest=True,
+          clone_by_rev=False, mirrors=None, bundles=None):
+    """Clones hg repo and places it at `dest`, replacing whatever else is
+    there.  The working copy will be empty.
+
+    If `revision` is set, only the specified revision and its ancestors will
+    be cloned.
+
+    If `update_dest` is set, then `dest` will be updated to `revision` if
+    set, otherwise to `branch`, otherwise to the head of default.
+
+    If `mirrors` is set, will try and clone from the mirrors before
+    cloning from `repo`.
+
+    If `bundles` is set, will try and download the bundle first and
+    unbundle it. If successful, will pull in new revisions from mirrors or
+    the master repo. If unbundling fails, will fall back to doing a regular
+    clone from mirrors or the master repo.
+
+    Regardless of how the repository ends up being cloned, the 'default' path
+    will point to `repo`.
+    """
+    if os.path.exists(dest):
+        remove_path(dest)
+
+    if bundles:
+        log.info("Attempting to initialize clone with bundles")
+        for bundle in bundles:
+            if os.path.exists(dest):
+                remove_path(dest)
+            init(dest)
+            log.info("Trying to use bundle %s", bundle)
+            try:
+                if not unbundle(bundle, dest):
+                    remove_path(dest)
+                    continue
+                adjust_paths(dest, default=repo)
+                # Now pull / update
+                return pull(repo, dest, update_dest=update_dest,
+                            mirrors=mirrors, revision=revision, branch=branch)
+            except Exception:
+                remove_path(dest)
+                log.exception("Problem unbundling/pulling from %s", bundle)
+                continue
+        else:
+            log.info("Using bundles failed; falling back to clone")
+
+    if mirrors:
+        log.info("Attempting to clone from mirrors")
+        for mirror in mirrors:
+            log.info("Cloning from %s", mirror)
+            try:
+                retval = clone(mirror, dest, branch, revision,
+                               update_dest=update_dest, clone_by_rev=clone_by_rev)
+                adjust_paths(dest, default=repo)
+                return retval
+            except:
+                log.exception("Problem cloning from mirror %s", mirror)
+                continue
+        else:
+            log.info("Pulling from mirrors failed; falling back to %s", repo)
+            # We may have a partial repo here; mercurial() copes with that
+            # We need to make sure our paths are correct though
+            if os.path.exists(os.path.join(dest, '.hg')):
+                adjust_paths(dest, default=repo)
+            return mercurial(repo, dest, branch, revision, autoPurge=True,
+                             update_dest=update_dest, clone_by_rev=clone_by_rev)
+
+    cmd = ['hg', 'clone']
+    if not update_dest:
+        cmd.append('-U')
+
+    if clone_by_rev:
+        if revision:
+            cmd.extend(['-r', revision])
+        elif branch:
+            # hg >= 1.6 supports -b branch for cloning
+            ver = hg_ver()
+            if ver >= (1, 6, 0):
+                cmd.extend(['-b', branch])
+
+    cmd.extend([repo, dest])
+    run_cmd(cmd)
+
+    if update_dest:
+        return update(dest, branch, revision)
+
+
+def common_args(revision=None, branch=None, ssh_username=None, ssh_key=None):
+    """Fill in common hg arguments, encapsulating logic checks that depend on
+       mercurial versions and provided arguments"""
+    args = []
+    if ssh_username or ssh_key:
+        opt = ['-e', 'ssh']
+        if ssh_username:
+            opt[1] += ' -l %s' % ssh_username
+        if ssh_key:
+            opt[1] += ' -i %s' % ssh_key
+        args.extend(opt)
+    if revision:
+        args.extend(['-r', revision])
+    elif branch:
+        if hg_ver() >= (1, 6, 0):
+            args.extend(['-b', branch])
+    return args
+
+
+def pull(repo, dest, update_dest=True, mirrors=None, **kwargs):
+    """Pulls changes from hg repo and places it in `dest`.
+
+    If `update_dest` is set, then `dest` will be updated to `revision` if
+    set, otherwise to `branch`, otherwise to the head of default.
+
+    If `mirrors` is set, will try and pull from the mirrors first before
+    `repo`."""
+
+    if mirrors:
+        for mirror in mirrors:
+            try:
+                return pull(mirror, dest, update_dest=update_dest, **kwargs)
+            except:
+                log.exception("Problem pulling from mirror %s", mirror)
+                continue
+        else:
+            log.info("Pulling from mirrors failed; falling back to %s", repo)
+
+    # Convert repo to an absolute path if it's a local repository
+    repo = _make_absolute(repo)
+    cmd = ['hg', 'pull']
+    # Don't pass -r to "hg pull", except when it's a valid HG revision.
+    # Pulling using tag names is dangerous: it uses the local .hgtags, so if
+    # the tag has moved on the remote side you won't pull the new revision the
+    # remote tag refers to.
+    pull_kwargs = kwargs.copy()
+    if 'revision' in pull_kwargs and \
+       not is_hg_cset(pull_kwargs['revision']):
+        del pull_kwargs['revision']
+
+    cmd.extend(common_args(**pull_kwargs))
+
+    cmd.append(repo)
+    run_cmd(cmd, cwd=dest)
+
+    if update_dest:
+        branch = None
+        if 'branch' in kwargs and kwargs['branch']:
+            branch = kwargs['branch']
+        revision = None
+        if 'revision' in kwargs and kwargs['revision']:
+            revision = kwargs['revision']
+        return update(dest, branch=branch, revision=revision)
+
+# Defines the places of attributes in the tuples returned by `out'
+REVISION, BRANCH = 0, 1
+
+
+def out(src, remote, **kwargs):
+    """Check for outgoing changesets present in a repo"""
+    cmd = ['hg', '-q', 'out', '--template', '{node} {branches}\n']
+    cmd.extend(common_args(**kwargs))
+    cmd.append(remote)
+    if os.path.exists(src):
+        try:
+            revs = []
+            for line in get_output(cmd, cwd=src).rstrip().split("\n"):
+                try:
+                    rev, branch = line.split()
+                # Mercurial displays no branch at all if the revision is on
+                # "default"
+                except ValueError:
+                    rev = line.rstrip()
+                    branch = "default"
+                revs.append((rev, branch))
+            return revs
+        except subprocess.CalledProcessError, inst:
+            # In some situations, some versions of Mercurial return "1"
+            # if no changes are found, so we need to ignore this return code
+            if inst.returncode == 1:
+                return []
+            raise
+
+
+def push(src, remote, push_new_branches=True, force=False, **kwargs):
+    cmd = ['hg', 'push']
+    cmd.extend(common_args(**kwargs))
+    if force:
+        cmd.append('-f')
+    if push_new_branches:
+        cmd.append('--new-branch')
+    cmd.append(remote)
+    run_cmd(cmd, cwd=src)
+
+
+def mercurial(repo, dest, branch=None, revision=None, update_dest=True,
+              shareBase=DefaultShareBase, allowUnsharedLocalClones=False,
+              clone_by_rev=False, mirrors=None, bundles=None, autoPurge=False):
+    """Makes sure that `dest` is has `revision` or `branch` checked out from
+    `repo`.
+
+    Do what it takes to make that happen, including possibly clobbering
+    dest.
+
+    If allowUnsharedLocalClones is True and we're trying to use the share
+    extension but fail, then we will be able to clone from the shared repo to
+    our destination.  If this is False, the default, then if we don't have the
+    share extension we will just clone from the remote repository.
+
+    If `clone_by_rev` is True, use 'hg clone -r <rev>' instead of 'hg clone'.
+    This is slower, but useful when cloning repos with lots of heads.
+
+    If `mirrors` is set, will try and use the mirrors before `repo`.
+
+    If `bundles` is set, will try and download the bundle first and
+    unbundle it instead of doing a full clone. If successful, will pull in
+    new revisions from mirrors or the master repo. If unbundling fails, will
+    fall back to doing a regular clone from mirrors or the master repo.
+    """
+    dest = os.path.abspath(dest)
+    if shareBase is DefaultShareBase:
+        shareBase = os.environ.get("HG_SHARE_BASE_DIR", None)
+
+    log.info("Reporting hg version in use")
+    cmd = ['hg', '-q', 'version']
+    run_cmd(cmd, cwd='.')
+
+    if shareBase:
+        # Check that 'hg share' works
+        try:
+            log.info("Checking if share extension works")
+            output = get_output(['hg', 'help', 'share'], dont_log=True)
+            if 'no commands defined' in output:
+                # Share extension is enabled, but not functional
+                log.info("Disabling sharing since share extension doesn't seem to work (1)")
+                shareBase = None
+            elif 'unknown command' in output:
+                # Share extension is disabled
+                log.info("Disabling sharing since share extension doesn't seem to work (2)")
+                shareBase = None
+        except subprocess.CalledProcessError:
+            # The command failed, so disable sharing
+            log.info("Disabling sharing since share extension doesn't seem to work (3)")
+            shareBase = None
+
+    # Check that our default path is correct
+    if os.path.exists(os.path.join(dest, '.hg')):
+        hgpath = path(dest, "default")
+
+        # Make sure that our default path is correct
+        if hgpath != _make_absolute(repo):
+            log.info("hg path isn't correct (%s should be %s); clobbering",
+                     hgpath, _make_absolute(repo))
+            remove_path(dest)
+
+    # If the working directory already exists and isn't using share we update
+    # the working directory directly from the repo, ignoring the sharing
+    # settings
+    if os.path.exists(dest):
+        if not os.path.exists(os.path.join(dest, ".hg")):
+            log.warning("%s doesn't appear to be a valid hg directory; clobbering", dest)
+            remove_path(dest)
+        elif not os.path.exists(os.path.join(dest, ".hg", "sharedpath")):
+            try:
+                if autoPurge:
+                    purge(dest)
+                return pull(repo, dest, update_dest=update_dest, branch=branch,
+                            revision=revision,
+                            mirrors=mirrors)
+            except subprocess.CalledProcessError:
+                log.warning("Error pulling changes into %s from %s; clobbering", dest, repo)
+                log.debug("Exception:", exc_info=True)
+                remove_path(dest)
+
+    # If that fails for any reason, and sharing is requested, we'll try to
+    # update the shared repository, and then update the working directory from
+    # that.
+    if shareBase:
+        sharedRepo = os.path.join(shareBase, get_repo_path(repo))
+        dest_sharedPath = os.path.join(dest, '.hg', 'sharedpath')
+
+        if os.path.exists(sharedRepo):
+            hgpath = path(sharedRepo, "default")
+
+            # Make sure that our default path is correct
+            if hgpath != _make_absolute(repo):
+                log.info("hg path isn't correct (%s should be %s); clobbering",
+                         hgpath, _make_absolute(repo))
+                # we need to clobber both the shared checkout and the dest,
+                # since hgrc needs to be in both places
+                remove_path(sharedRepo)
+                remove_path(dest)
+
+        if os.path.exists(dest_sharedPath):
+            # Make sure that the sharedpath points to sharedRepo
+            dest_sharedPath_data = os.path.normpath(
+                open(dest_sharedPath).read())
+            norm_sharedRepo = os.path.normpath(os.path.join(sharedRepo, '.hg'))
+            if dest_sharedPath_data != norm_sharedRepo:
+                # Clobber!
+                log.info("We're currently shared from %s, but are being requested to pull from %s (%s); clobbering",
+                         dest_sharedPath_data, repo, norm_sharedRepo)
+                remove_path(dest)
+
+        try:
+            log.info("Updating shared repo")
+            mercurial(repo, sharedRepo, branch=branch, revision=revision,
+                      update_dest=False, shareBase=None, clone_by_rev=clone_by_rev,
+                      mirrors=mirrors, bundles=bundles, autoPurge=False)
+            if os.path.exists(dest):
+                if autoPurge:
+                    purge(dest)
+                return update(dest, branch=branch, revision=revision)
+
+            try:
+                log.info("Trying to share %s to %s", sharedRepo, dest)
+                return share(sharedRepo, dest, branch=branch, revision=revision)
+            except subprocess.CalledProcessError:
+                if not allowUnsharedLocalClones:
+                    # Re-raise the exception so it gets caught below.
+                    # We'll then clobber dest, and clone from original repo
+                    raise
+
+                log.warning("Error calling hg share from %s to %s;"
+                            "falling back to normal clone from shared repo",
+                            sharedRepo, dest)
+                # Do a full local clone first, and then update to the
+                # revision we want
+                # This lets us use hardlinks for the local clone if the OS
+                # supports it
+                clone(sharedRepo, dest, update_dest=False,
+                      mirrors=mirrors, bundles=bundles)
+                return update(dest, branch=branch, revision=revision)
+        except subprocess.CalledProcessError:
+            log.warning(
+                "Error updating %s from sharedRepo (%s): ", dest, sharedRepo)
+            log.debug("Exception:", exc_info=True)
+            remove_path(dest)
+    # end if shareBase
+
+    if not os.path.exists(os.path.dirname(dest)):
+        os.makedirs(os.path.dirname(dest))
+
+    # Share isn't available or has failed, clone directly from the source
+    return clone(repo, dest, branch, revision,
+                 update_dest=update_dest, mirrors=mirrors,
+                 bundles=bundles, clone_by_rev=clone_by_rev)
+
+
+def apply_and_push(localrepo, remote, changer, max_attempts=10,
+                   ssh_username=None, ssh_key=None, force=False):
+    """This function calls `changer' to make changes to the repo, and tries
+       its hardest to get them to the origin repo. `changer' must be a
+       callable object that receives two arguments: the directory of the local
+       repository, and the attempt number. This function will push ALL
+       changesets missing from remote."""
+    assert callable(changer)
+    branch = get_branch(localrepo)
+    changer(localrepo, 1)
+    for n in range(1, max_attempts + 1):
+        new_revs = []
+        try:
+            new_revs = out(src=localrepo, remote=remote,
+                           ssh_username=ssh_username,
+                           ssh_key=ssh_key)
+            if len(new_revs) < 1:
+                raise HgUtilError("No revs to push")
+            push(src=localrepo, remote=remote, ssh_username=ssh_username,
+                 ssh_key=ssh_key, force=force)
+            return
+        except subprocess.CalledProcessError, e:
+            log.debug("Hit error when trying to push: %s" % str(e))
+            if n == max_attempts:
+                log.debug("Tried %d times, giving up" % max_attempts)
+                for r in reversed(new_revs):
+                    run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',
+                             r[REVISION]], cwd=localrepo)
+                raise HgUtilError("Failed to push")
+            pull(remote, localrepo, update_dest=False,
+                 ssh_username=ssh_username, ssh_key=ssh_key)
+            # After we successfully rebase or strip away heads the push is
+            # is attempted again at the start of the loop
+            try:
+                run_cmd(['hg', '--config', 'ui.merge=internal:merge',
+                         'rebase'], cwd=localrepo)
+            except subprocess.CalledProcessError, e:
+                log.debug("Failed to rebase: %s" % str(e))
+                update(localrepo, branch=branch)
+                for r in reversed(new_revs):
+                    run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',
+                             r[REVISION]], cwd=localrepo)
+                changer(localrepo, n + 1)
+
+
+def share(source, dest, branch=None, revision=None):
+    """Creates a new working directory in "dest" that shares history with
+       "source" using Mercurial's share extension"""
+    run_cmd(['hg', 'share', '-U', source, dest])
+    return update(dest, branch=branch, revision=revision)
+
+
+def cleanOutgoingRevs(reponame, remote, username, sshKey):
+    outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
+                                          ssh_username=username,
+                                          ssh_key=sshKey))
+    for r in reversed(outgoingRevs):
+        run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',
+                 r[REVISION]], cwd=reponame)
+
+
+def path(src, name='default'):
+    """Returns the remote path associated with "name" """
+    try:
+        return get_output(['hg', 'path', name], cwd=src).strip()
+    except subprocess.CalledProcessError:
+        return None
+
+
+def init(dest):
+    """Initializes an empty repo in `dest`"""
+    run_cmd(['hg', 'init', dest])
+
+
+def unbundle(bundle, dest):
+    """Unbundles the bundle located at `bundle` into `dest`.
+
+    `bundle` can be a local file or remote url."""
+    try:
+        get_output(['hg', 'unbundle', bundle], cwd=dest, include_stderr=True)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
+def adjust_paths(dest, **paths):
+    """Adjusts paths in `dest`/.hg/hgrc so that names in `paths` are set to
+    paths[name].
+
+    Note that any comments in the hgrc will be lost if changes are made to the
+    file."""
+    hgrc = os.path.join(dest, '.hg', 'hgrc')
+    config = RawConfigParser()
+    config.read(hgrc)
+
+    if not config.has_section('paths'):
+        config.add_section('paths')
+
+    changed = False
+    for path_name, path_value in paths.items():
+        if (not config.has_option('paths', path_name) or
+                config.get('paths', path_name) != path_value):
+            changed = True
+            config.set('paths', path_name, path_value)
+
+    if changed:
+        config.write(open(hgrc, 'w'))
+
+
+def commit(dest, msg, user=None):
+    cmd = ['hg', 'commit', '-m', msg]
+    if user:
+        cmd.extend(['-u', user])
+    run_cmd(cmd, cwd=dest)
+    return get_revision(dest)
+
+
+def tag(dest, tags, user=None, msg=None, rev=None, force=None):
+    cmd = ['hg', 'tag']
+    if user:
+        cmd.extend(['-u', user])
+    if msg:
+        cmd.extend(['-m', msg])
+    if rev:
+        cmd.extend(['-r', rev])
+    if force:
+        cmd.append('-f')
+    cmd.extend(tags)
+    run_cmd(cmd, cwd=dest)
+    return get_revision(dest)