Bug 699407: hgtool support for bundles and mirrors. r=bhearsum
authorChris AtLee <catlee@mozilla.com>
Tue, 08 Nov 2011 14:01:26 -0500
changeset 1951 437cce872cb7
parent 1950 a1b77922b0b7
child 1952 24afe4885a7d
push id1375
push usercatlee@mozilla.com
push date2011-11-08 19:03 +0000
reviewersbhearsum
bugs699407, 691467, 687064
Bug 699407: hgtool support for bundles and mirrors. r=bhearsum

Also fixes bugs 691467, 687064
buildfarm/utils/hgtool.py
buildfarm/utils/retry.py
lib/python/mozilla_buildtools/test/test_util_hg.py
lib/python/util/hg.py
--- a/buildfarm/utils/hgtool.py
+++ b/buildfarm/utils/hgtool.py
@@ -5,44 +5,55 @@
 Tool to do safe operations with hg.
 
 revision/branch on commandline will override those in props-file"""
 
 # Import snippet to find tools lib
 import os, sys
 sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib/python"))
 
-from util.hg import mercurial, share, out, remove_path
+from util.hg import mercurial, out, remove_path
 
 if __name__ == '__main__':
     from optparse import OptionParser
     import logging
 
     parser = OptionParser(__doc__)
     parser.set_defaults(
             revision=os.environ.get('HG_REV'),
             branch=os.environ.get('HG_BRANCH', 'default'),
             outgoing=False,
             propsfile=os.environ.get('PROPERTIES_FILE'),
             tbox=bool(os.environ.get('PROPERTIES_FILE')),
             loglevel=logging.INFO,
-            shared_dir=os.environ.get('HG_SHARE_BASE_DIR')
+            shared_dir=os.environ.get('HG_SHARE_BASE_DIR'),
+            clone_by_rev=False,
+            mirrors=None,
+            bundles=None,
             )
     parser.add_option("-r", "--rev", dest="revision", help="which revision to update to")
     parser.add_option("-b", "--branch", dest="branch", help="which branch to update to")
     parser.add_option("-p", "--props-file", dest="propsfile",
         help="build json file containing revision information")
     parser.add_option("--tbox", dest="tbox", action="store_true",
         help="output TinderboxPrint messages")
     parser.add_option("--no-tbox", dest="tbox", action="store_false",
         help="don't output TinderboxPrint messages")
     parser.add_option("-s", "--shared-dir", dest="shared_dir",
         help="clone to a shared directory")
     parser.add_option("--check-outgoing", dest="outgoing", action="store_true",
         help="check for and clobber outgoing changesets")
+    parser.add_option("--clone-by-revision", dest="clone_by_rev", action="store_true",
+        help="do initial clone with -r <rev> instead of cloning the entire repo. "
+             "This is slower but is useful when cloning repositories with many "
+             "heads which may timeout otherwise.")
+    parser.add_option("--mirror", dest="mirrors", action="append",
+        help="add a mirror to try cloning/pulling from before repo")
+    parser.add_option("--bundle", dest="bundles", action="append",
+        help="add a bundle to try downloading/unbundling from before doing a full clone")
 
     options, args = parser.parse_args()
 
     logging.basicConfig(level=options.loglevel, format="%(message)s")
 
     if len(args) not in (1, 2):
         parser.error("Invalid number of arguments")
 
@@ -67,17 +78,20 @@ if __name__ == '__main__':
     #look for and clobber outgoing changesets
     if options.outgoing:
         if out(dest, repo):
             remove_path(dest)
         if options.shared_dir and out(options.shared_dir, repo):
             remove_path(options.shared_dir)
 
     got_revision = mercurial(repo, dest, options.branch, options.revision,
-                             shareBase=options.shared_dir)
+                             shareBase=options.shared_dir,
+                             clone_by_rev=options.clone_by_rev,
+                             mirrors=options.mirrors,
+                             bundles=options.bundles)
 
     if options.tbox:
         if repo.startswith("http"):
             url = "%s/rev/%s" % (repo, got_revision)
             print "TinderboxPrint: <a href=\"%(url)s\">revision: %(got_revision)s</a>" % locals()
         else:
             print "TinderboxPrint: revision: %s" % got_revision
     else:
--- a/buildfarm/utils/retry.py
+++ b/buildfarm/utils/retry.py
@@ -30,27 +30,34 @@ def search_output(f, regexp, fail_if_mat
 
 class RunWithTimeoutException(Exception):
     def __init__(self, rc, **kwargs):
         Exception.__init__(self, **kwargs)
         self.rc = rc
 
 def run_with_timeout(cmd, timeout, stdout_regexp=None, stderr_regexp=None,
                      fail_if_match=False, print_output=True):
-    stdout = TemporaryFile()
-    stderr = TemporaryFile()
+    if stdout_regexp:
+        stdout = TemporaryFile()
+    else:
+        stdout = None
+    if stderr_regexp:
+        stderr = TemporaryFile()
+    else:
+        stderr = None
     proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
     start_time = time.time()
     log.info("Executing: %s", cmd)
     while True:
         rc = proc.poll()
         if rc is not None:
             log.debug("Process returned %s", rc)
-            if print_output:
+            if print_output and stdout:
                 print "Process stdio:\n%s" % read_file(stdout)
+            if print_output and stderr:
                 print "Process stderr:\n%s" % read_file(stderr)
             if rc == 0:
                 if stdout_regexp and not \
                    search_output(stdout, stdout_regexp, fail_if_match):
                     raise RunWithTimeoutException("%s found in stdout, failing" % fail_if_match, -1)
                 if stderr_regexp and not \
                    search_output(stderr, stderr_regexp, fail_if_match):
                     raise RunWithTimeoutException("%s found in stderr, failing" % fail_if_match, -1)
--- a/lib/python/mozilla_buildtools/test/test_util_hg.py
+++ b/lib/python/mozilla_buildtools/test/test_util_hg.py
@@ -1,17 +1,18 @@
 import unittest
 import tempfile
 import shutil
 import os
 import subprocess
 
+import util.hg as hg
 from util.hg import clone, pull, update, hg_ver, mercurial, _make_absolute, \
   share, push, apply_and_push, HgUtilError, make_hg_url, get_branch, \
-  get_branches, path
+  get_branches, path, init, unbundle, adjust_paths
 from util.commands import run_cmd, get_output
 
 def getRevisions(dest):
     retval = []
     for rev in get_output(['hg', 'log', '-R', dest, '--template', '{node|short}\n']).split('\n'):
         rev = rev.strip()
         if not rev:
             continue
@@ -72,69 +73,73 @@ class TestHg(unittest.TestCase):
         self.failUnless(not os.path.exists(os.path.join(self.wc, 'test.txt')))
 
     def testCloneUpdate(self):
         rev = clone(self.repodir, self.wc, update_dest=True)
         self.assertEquals(rev, self.revisions[0])
 
     def testCloneBranch(self):
         clone(self.repodir, self.wc, branch='branch2',
-                update_dest=False)
+                update_dest=False, clone_by_rev=True)
         # On hg 1.6, we should only have a subset of the revisions
         if hg_ver() >= (1,6,0):
             self.assertEquals(self.revisions[1:],
                     getRevisions(self.wc))
         else:
             self.assertEquals(self.revisions,
                     getRevisions(self.wc))
 
     def testCloneUpdateBranch(self):
         rev = clone(self.repodir, os.path.join(self.tmpdir, 'wc'),
-                branch="branch2", update_dest=True)
+                branch="branch2", update_dest=True, clone_by_rev=True)
         self.assertEquals(rev, self.revisions[1], self.revisions)
 
     def testCloneRevision(self):
         clone(self.repodir, self.wc,
-                revision=self.revisions[0], update_dest=False)
+                revision=self.revisions[0], update_dest=False,
+                clone_by_rev=True)
         # We'll only get a subset of the revisions
         self.assertEquals(self.revisions[:1] + self.revisions[2:],
                 getRevisions(self.wc))
 
     def testUpdateRevision(self):
         rev = clone(self.repodir, self.wc, update_dest=False)
         self.assertEquals(rev, None)
 
         rev = update(self.wc, revision=self.revisions[1])
         self.assertEquals(rev, self.revisions[1])
 
     def testPull(self):
         # Clone just the first rev
-        clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False)
+        clone(self.repodir, self.wc, revision=self.revisions[-1],
+                update_dest=False, clone_by_rev=True)
         self.assertEquals(getRevisions(self.wc), self.revisions[-1:])
 
         # Now pull in new changes
         rev = pull(self.repodir, self.wc, update_dest=False)
         self.assertEquals(rev, None)
         self.assertEquals(getRevisions(self.wc), self.revisions)
 
     def testPullRevision(self):
         # Clone just the first rev
-        clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False)
+        clone(self.repodir, self.wc, revision=self.revisions[-1],
+                update_dest=False, clone_by_rev=True)
         self.assertEquals(getRevisions(self.wc), self.revisions[-1:])
 
         # Now pull in just the last revision
         rev = pull(self.repodir, self.wc, revision=self.revisions[0], update_dest=False)
         self.assertEquals(rev, None)
 
         # We'll be missing the middle revision (on another branch)
         self.assertEquals(getRevisions(self.wc), self.revisions[:1] + self.revisions[2:])
 
     def testPullBranch(self):
         # Clone just the first rev
-        clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False)
+        clone(self.repodir, self.wc, revision=self.revisions[-1],
+                update_dest=False, clone_by_rev=True)
         self.assertEquals(getRevisions(self.wc), self.revisions[-1:])
 
         # Now pull in the other branch
         rev = pull(self.repodir, self.wc, branch="branch2", update_dest=False)
         self.assertEquals(rev, None)
 
         # On hg 1.6, we'll be missing the last revision (on another branch)
         if hg_ver() >= (1,6,0):
@@ -187,37 +192,37 @@ class TestHg(unittest.TestCase):
 
         # Try and update our working copy
         mercurial(self.repodir, self.wc, shareBase=shareBase)
 
         self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
         self.assertNotEqual(old_revs, getRevisions(self.wc))
 
     def testPush(self):
-        clone(self.repodir, self.wc, revision=self.revisions[-2])
+        clone(self.repodir, self.wc, revision=self.revisions[-2], clone_by_rev=True)
         push(src=self.repodir, remote=self.wc)
         self.assertEquals(getRevisions(self.wc), self.revisions)
 
     def testPushWithBranch(self):
-        clone(self.repodir, self.wc, revision=self.revisions[-1])
+        clone(self.repodir, self.wc, revision=self.revisions[-1], clone_by_rev=True)
         push(src=self.repodir, remote=self.wc, branch='branch2')
         push(src=self.repodir, remote=self.wc, branch='default')
         self.assertEquals(getRevisions(self.wc), self.revisions)
 
     def testPushWithRevision(self):
-        clone(self.repodir, self.wc, revision=self.revisions[-2])
+        clone(self.repodir, self.wc, revision=self.revisions[-2], clone_by_rev=True)
         push(src=self.repodir, remote=self.wc, revision=self.revisions[-1])
         self.assertEquals(getRevisions(self.wc), self.revisions[-2:])
 
     def testMercurial(self):
         rev = mercurial(self.repodir, self.wc)
         self.assertEquals(rev, self.revisions[0])
 
     def testPushNewBranchesNotAllowed(self):
-        clone(self.repodir, self.wc, revision=self.revisions[0])
+        clone(self.repodir, self.wc, revision=self.revisions[0], clone_by_rev=True)
         self.assertRaises(Exception, push, self.repodir, self.wc,
                           push_new_branches=False)
 
     def testMercurialWithNewShare(self):
         shareBase = os.path.join(self.tmpdir, 'share')
         sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
         os.mkdir(shareBase)
         mercurial(self.repodir, self.wc, shareBase=shareBase)
@@ -419,17 +424,17 @@ class TestHg(unittest.TestCase):
             pass
         self.assertRaises(HgUtilError, apply_and_push, self.wc, self.repodir, c)
 
     def testPath(self):
         clone(self.repodir, self.wc)
         p = path(self.wc)
         self.assertEquals(p, self.repodir)
 
-    def testBustedHgrc(self):
+    def testBustedHgrcWithShare(self):
         # Test that we can recover from hgrc being lost
         shareBase = os.path.join(self.tmpdir, 'share')
         sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
         os.mkdir(shareBase)
         mercurial(self.repodir, self.wc, shareBase=shareBase)
 
         # Delete .hg/hgrc
         for d in sharerepo, self.wc:
@@ -440,8 +445,351 @@ class TestHg(unittest.TestCase):
         p = path(self.wc)
         self.assertEquals(p, None)
 
         # cloning again should fix this up
         mercurial(self.repodir, self.wc, shareBase=shareBase)
 
         p = path(self.wc)
         self.assertEquals(p, self.repodir)
+
+    def testBustedHgrc(self):
+        # Test that we can recover from hgrc being lost
+        mercurial(self.repodir, self.wc)
+
+        # Delete .hg/hgrc
+        os.unlink(os.path.join(self.wc, '.hg', 'hgrc'))
+
+        # path is busted now
+        p = path(self.wc)
+        self.assertEquals(p, None)
+
+        # cloning again should fix this up
+        mercurial(self.repodir, self.wc)
+
+        p = path(self.wc)
+        self.assertEquals(p, self.repodir)
+
+    def testInit(self):
+        tmpdir = os.path.join(self.tmpdir, 'new')
+        self.assertEquals(False, os.path.exists(tmpdir))
+        init(tmpdir)
+        self.assertEquals(True, os.path.exists(tmpdir))
+        self.assertEquals(True, os.path.exists(os.path.join(tmpdir, '.hg')))
+
+    def testUnbundle(self):
+        # First create the bundle
+        bundle = os.path.join(self.tmpdir, 'bundle')
+        run_cmd(['hg', 'bundle', '-a', bundle], cwd=self.repodir)
+
+        # Now unbundle it in a new place
+        newdir = os.path.join(self.tmpdir, 'new')
+        init(newdir)
+        unbundle(bundle, newdir)
+
+        self.assertEquals(self.revisions, getRevisions(newdir))
+
+    def testCloneWithBundle(self):
+        # First create the bundle
+        bundle = os.path.join(self.tmpdir, 'bundle')
+        run_cmd(['hg', 'bundle', '-a', bundle], cwd=self.repodir)
+
+        # Wrap unbundle so we can tell if it got called
+        orig_unbundle = unbundle
+        try:
+            called = []
+            def new_unbundle(*args, **kwargs):
+                called.append(True)
+                return orig_unbundle(*args, **kwargs)
+            hg.unbundle = new_unbundle
+
+            # Now clone it using the bundle
+            clone(self.repodir, self.wc, bundles=[bundle])
+            self.assertEquals(self.revisions, getRevisions(self.wc))
+            self.assertEquals(called, [True])
+        finally:
+            hg.unbundle = orig_unbundle
+
+    def testCloneWithUnrelatedBundle(self):
+        # First create the bundle
+        bundle = os.path.join(self.tmpdir, 'bundle')
+        run_cmd(['hg', 'bundle', '-a', bundle], cwd=self.repodir)
+
+        # Create an unrelated repo
+        repo2 = os.path.join(self.tmpdir, 'repo2')
+        run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__),
+            repo2])
+
+        self.assertNotEqual(self.revisions, getRevisions(repo2))
+
+        # Clone repo2 using the unrelated bundle
+        clone(repo2, self.wc, bundles=[bundle])
+
+        # Make sure we don't have unrelated revisions
+        self.assertEquals(getRevisions(repo2), getRevisions(self.wc))
+        self.assertEquals(set(),
+                set(getRevisions(self.repodir)).intersection(set(getRevisions(self.wc))))
+
+    def testCloneWithBadBundle(self):
+        # First create the bad bundle
+        bundle = os.path.join(self.tmpdir, 'bundle')
+        open(bundle, 'w').write('ruh oh!')
+
+        # Wrap unbundle so we can tell if it got called
+        orig_unbundle = unbundle
+        try:
+            called = []
+            def new_unbundle(*args, **kwargs):
+                called.append(True)
+                return orig_unbundle(*args, **kwargs)
+            hg.unbundle = new_unbundle
+
+            # Now clone it using the bundle
+            clone(self.repodir, self.wc, bundles=[bundle])
+            self.assertEquals(self.revisions, getRevisions(self.wc))
+            self.assertEquals(called, [True])
+        finally:
+            hg.unbundle = orig_unbundle
+
+    def testCloneWithBundleMissingRevs(self):
+        # First create the bundle
+        bundle = os.path.join(self.tmpdir, 'bundle')
+        run_cmd(['hg', 'bundle', '-a', bundle], cwd=self.repodir)
+
+        # Create a commit
+        open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
+        run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
+
+        # Wrap unbundle so we can tell if it got called
+        orig_unbundle = unbundle
+        try:
+            called = []
+            def new_unbundle(*args, **kwargs):
+                called.append(True)
+                return orig_unbundle(*args, **kwargs)
+            hg.unbundle = new_unbundle
+
+            # Now clone it using the bundle
+            clone(self.repodir, self.wc, bundles=[bundle])
+            self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
+            self.assertEquals(called, [True])
+        finally:
+            hg.unbundle = orig_unbundle
+
+    def testCloneWithMirror(self):
+        mirror = os.path.join(self.tmpdir, 'repo2')
+        clone(self.repodir, mirror)
+
+        # Create a commit in the original repo
+        open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
+        run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
+
+        # Now clone from the mirror
+        clone(self.repodir, self.wc, mirrors=[mirror])
+
+        # We'll be missing the new revision from repodir
+        self.assertNotEquals(getRevisions(self.repodir), getRevisions(self.wc))
+        # But we should have everything from the mirror
+        self.assertEquals(getRevisions(mirror), getRevisions(self.wc))
+        # Our default path should point to the original repo though.
+        self.assertEquals(self.repodir, path(self.wc))
+
+    def testCloneWithBadMirror(self):
+        mirror = os.path.join(self.tmpdir, 'repo2')
+
+        # Now clone from the mirror
+        clone(self.repodir, self.wc, mirrors=[mirror])
+
+        # We still end up with a valid repo
+        self.assertEquals(self.revisions, getRevisions(self.wc))
+        self.assertEquals(self.repodir, path(self.wc))
+
+    def testCloneWithOneBadMirror(self):
+        mirror1 = os.path.join(self.tmpdir, 'mirror1')
+        mirror2 = os.path.join(self.tmpdir, 'mirror2')
+
+        # Mirror 2 is ok
+        clone(self.repodir, mirror2)
+
+        # Create a commit in the original repo
+        open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
+        run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
+
+        # Now clone from the mirror
+        clone(self.repodir, self.wc, mirrors=[mirror1, mirror2])
+
+        # We'll be missing the new revision from repodir
+        self.assertNotEquals(getRevisions(self.repodir), getRevisions(self.wc))
+        # But we should have everything from the mirror
+        self.assertEquals(getRevisions(mirror2), getRevisions(self.wc))
+        # Our default path should point to the original repo though.
+        self.assertEquals(self.repodir, path(self.wc))
+
+    def testCloneWithRevAndMirror(self):
+        mirror = os.path.join(self.tmpdir, 'repo2')
+        clone(self.repodir, mirror)
+
+        # Create a commit in the original repo
+        open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
+        run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
+
+        # Now clone from the mirror
+        revisions = getRevisions(self.repodir)
+        clone(self.repodir, self.wc, revision=revisions[0],
+                mirrors=[mirror], clone_by_rev=True)
+
+        # We'll be missing the middle revision (on another branch)
+        self.assertEquals(revisions[:2] + revisions[3:], getRevisions(self.wc))
+        # But not from the mirror
+        self.assertNotEquals(getRevisions(mirror), getRevisions(self.wc))
+        # Our default path should point to the original repo though.
+        self.assertEquals(self.repodir, path(self.wc))
+
+    def testPullWithMirror(self):
+        mirror = os.path.join(self.tmpdir, 'repo2')
+        clone(self.repodir, mirror)
+
+        # Create a new commit in the mirror repo
+        open(os.path.join(mirror, 'test.txt'), 'w').write('hello!')
+        run_cmd(['hg', 'add', 'test.txt'], cwd=mirror)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=mirror)
+
+        # Now clone from the original
+        clone(self.repodir, self.wc)
+
+        # Pull using the mirror
+        pull(self.repodir, self.wc, mirrors=[mirror])
+
+        self.assertEquals(getRevisions(self.wc), getRevisions(mirror))
+
+        # Our default path should point to the original repo
+        self.assertEquals(self.repodir, path(self.wc))
+
+    def testPullWithBadMirror(self):
+        mirror = os.path.join(self.tmpdir, 'repo2')
+
+        # Now clone from the original
+        clone(self.repodir, self.wc)
+
+        # Create a new commit in the original repo
+        open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
+        run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
+
+        # Pull using the mirror
+        pull(self.repodir, self.wc, mirrors=[mirror])
+
+        self.assertEquals(getRevisions(self.wc), getRevisions(self.repodir))
+
+        # Our default path should point to the original repo
+        self.assertEquals(self.repodir, path(self.wc))
+
+    def testPullWithUnrelatedMirror(self):
+        mirror = os.path.join(self.tmpdir, 'repo2')
+        run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), mirror])
+
+        # Now clone from the original
+        clone(self.repodir, self.wc)
+
+        # Create a new commit in the original repo
+        open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
+        run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
+
+        # Pull using the mirror
+        pull(self.repodir, self.wc, mirrors=[mirror])
+
+        self.assertEquals(getRevisions(self.wc), getRevisions(self.repodir))
+        # We shouldn't have anything from the unrelated mirror
+        self.assertEquals(set(),
+                set(getRevisions(mirror)).intersection(set(getRevisions(self.wc))))
+
+        # Our default path should point to the original repo
+        self.assertEquals(self.repodir, path(self.wc))
+
+    def testMercurialWithShareAndBundle(self):
+        # First create the bundle
+        bundle = os.path.join(self.tmpdir, 'bundle')
+        run_cmd(['hg', 'bundle', '-a', bundle], cwd=self.repodir)
+
+        # Create a commit
+        open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
+        run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
+
+        # Wrap unbundle so we can tell if it got called
+        orig_unbundle = unbundle
+        try:
+            called = []
+            def new_unbundle(*args, **kwargs):
+                called.append(True)
+                return orig_unbundle(*args, **kwargs)
+            hg.unbundle = new_unbundle
+
+            shareBase = os.path.join(self.tmpdir, 'share')
+            sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
+            os.mkdir(shareBase)
+            mercurial(self.repodir, self.wc, shareBase=shareBase, bundles=[bundle])
+
+            self.assertEquals(called, [True])
+            self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
+            self.assertEquals(getRevisions(self.repodir), getRevisions(sharerepo))
+        finally:
+            hg.unbundle = orig_unbundle
+
+    def testMercurialWithShareAndMirror(self):
+        # First create the mirror
+        mirror = os.path.join(self.tmpdir, 'repo2')
+        clone(self.repodir, mirror)
+
+        # Create a commit
+        open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
+        run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
+
+        shareBase = os.path.join(self.tmpdir, 'share')
+        sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
+        os.mkdir(shareBase)
+        mercurial(self.repodir, self.wc, shareBase=shareBase, mirrors=[mirror])
+
+        # Since we used the mirror, we should be missing a commit
+        self.assertNotEquals(getRevisions(self.repodir), getRevisions(self.wc))
+        self.assertNotEquals(getRevisions(self.repodir), getRevisions(sharerepo))
+        self.assertEquals(getRevisions(mirror), getRevisions(self.wc))
+
+    def testMercurialByRevWithShareAndMirror(self):
+        # First create the mirror
+        mirror = os.path.join(self.tmpdir, 'repo2')
+        clone(self.repodir, mirror)
+
+        shareBase = os.path.join(self.tmpdir, 'share')
+        sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
+        os.mkdir(shareBase)
+        mercurial(self.repodir, self.wc, shareBase=shareBase, mirrors=[mirror], clone_by_rev=True, revision=self.revisions[-1])
+
+        # We should only have the one revision
+        self.assertEquals(getRevisions(sharerepo), self.revisions[-1:])
+        self.assertEquals(getRevisions(self.wc), self.revisions[-1:])
+
+    def testAdjustPaths(self):
+        mercurial(self.repodir, self.wc)
+
+        # Make sure our default path is correct
+        self.assertEquals(path(self.wc), self.repodir)
+
+        # Add a comment, make sure it's still there if we don't change
+        # anything
+        hgrc = os.path.join(self.wc, '.hg', 'hgrc')
+        open(hgrc, 'a').write("# Hello world")
+        adjust_paths(self.wc, default=self.repodir)
+        self.assert_("Hello world" in open(hgrc).read())
+
+        # Add a path, and the comment goes away
+        adjust_paths(self.wc, test=self.repodir)
+        self.assert_("Hello world" not in open(hgrc).read())
+
+        # Make sure our paths are correct
+        self.assertEquals(path(self.wc), self.repodir)
+        self.assertEquals(path(self.wc, 'test'), self.repodir)
--- a/lib/python/util/hg.py
+++ b/lib/python/util/hg.py
@@ -1,11 +1,12 @@
 """Functions for interacting with hg"""
 import os, re, subprocess
 from urlparse import urlsplit
+from ConfigParser import RawConfigParser
 
 from util.commands import run_cmd, get_output, remove_path
 from util.retry import retry
 
 import logging
 log = logging.getLogger(__name__)
 
 class DefaultShareBase:
@@ -92,39 +93,91 @@ def update(dest, branch=None, revision=N
 
         # If this is different, checkout the other branch
         if branch and branch != local_branch:
             cmd.append(branch)
 
         run_cmd(cmd, cwd=dest)
     return get_revision(dest)
 
-def clone(repo, dest, branch=None, revision=None, update_dest=True):
+def clone(repo, dest, branch=None, revision=None, update_dest=True,
+        clone_by_rev=False, mirrors=None, bundles=None):
     """Clones hg repo and places it at `dest`, replacing whatever else is
     there.  The working copy will be empty.
 
-    If `revision` is set, only the specified revision and its ancestors will be
-    cloned.
+    If `revision` is set, only the specified revision and its ancestors will
+    be cloned.
 
-    If `update_dest` is set, then `dest` will be updated to `revision` if set,
-    otherwise to `branch`, otherwise to the head of default."""
+    If `update_dest` is set, then `dest` will be updated to `revision` if
+    set, otherwise to `branch`, otherwise to the head of default.
+
+    If `mirrors` is set, will try and clone from the mirrors before
+    cloning from `repo`.
+
+    If `bundles` is set, will try and download the bundle first and
+    unbundle it. If successful, will pull in new revisions from mirrors or
+    the master repo. If unbundling fails, will fall back to doing a regular
+    clone from mirrors or the master repo.
+
+    Regardless of how the repository ends up being cloned, the 'default' path
+    will point to `repo`.
+    """
     if os.path.exists(dest):
         remove_path(dest)
 
+    if bundles:
+        log.info("Attempting to initialize clone with bundles")
+        init(dest)
+
+        for bundle in bundles:
+            log.info("Trying to use bundle %s", bundle)
+            try:
+                unbundle(bundle, dest)
+                # Now pull / update
+                return pull(repo, dest, update_dest=update_dest, mirrors=mirrors)
+            except:
+                remove_path(dest)
+                log.exception("Problem unbundling/pulling from %s", bundle)
+                continue
+        else:
+            log.info("Using bundles failed; falling back to clone")
+
+    if mirrors:
+        log.info("Attempting to clone from mirrors")
+        for mirror in mirrors:
+            log.info("Cloning from %s", mirror)
+            try:
+                retval = clone(mirror, dest, branch, revision,
+                        update_dest=update_dest, clone_by_rev=clone_by_rev)
+                adjust_paths(dest, default=repo)
+                return retval
+            except:
+                log.exception("Problem cloning from mirror %s", mirror)
+                continue
+        else:
+            log.info("Pulling from mirrors failed; falling back to %s", repo)
+            # We may have a partial repo here; mercurial() copes with that
+            # We need to make sure our paths are correct though
+            if os.path.exists(os.path.join(dest, '.hg')):
+                adjust_paths(dest, default=repo)
+            return mercurial(repo, dest, branch, revision,
+                    update_dest=update_dest, clone_by_rev=clone_by_rev)
+
     cmd = ['hg', 'clone']
     if not update_dest:
         cmd.append('-U')
 
-    if revision:
-        cmd.extend(['-r', revision])
-    elif branch:
-        # hg >= 1.6 supports -b branch for cloning
-        ver = hg_ver()
-        if ver >= (1, 6, 0):
-            cmd.extend(['-b', branch])
+    if clone_by_rev:
+        if revision:
+            cmd.extend(['-r', revision])
+        elif branch:
+            # hg >= 1.6 supports -b branch for cloning
+            ver = hg_ver()
+            if ver >= (1, 6, 0):
+                cmd.extend(['-b', branch])
 
     cmd.extend([repo, dest])
     run_cmd(cmd)
 
     if update_dest:
         return update(dest, branch, revision)
 
 def common_args(revision=None, branch=None, ssh_username=None, ssh_key=None):
@@ -140,28 +193,44 @@ def common_args(revision=None, branch=No
         args.extend(opt)
     if revision:
         args.extend(['-r', revision])
     elif branch:
         if hg_ver() >= (1, 6, 0):
             args.extend(['-b', branch])
     return args
 
-def pull(repo, dest, update_dest=True, **kwargs):
+def pull(repo, dest, update_dest=True, mirrors=None, **kwargs):
     """Pulls changes from hg repo and places it in `dest`.
 
-    If `revision` is set, only the specified revision and its ancestors will be
-    pulled.
+    If `revision` is set, only the specified revision and its ancestors will
+    be pulled.
 
-    If `update_dest` is set, then `dest` will be updated to `revision` if set,
-    otherwise to `branch`, otherwise to the head of default.  """
+    If `update_dest` is set, then `dest` will be updated to `revision` if
+    set, otherwise to `branch`, otherwise to the head of default.
+
+    If `mirrors` is set, will try and pull from the mirrors first before
+    `repo`."""
+
+    if mirrors:
+        for mirror in mirrors:
+            try:
+                retval = pull(mirror, dest, update_dest=update_dest, **kwargs)
+                return retval
+            except:
+                log.exception("Problem pulling from mirror %s", mirror)
+                continue
+        else:
+            log.info("Pulling from mirrors failed; falling back to %s", repo)
+
     # Convert repo to an absolute path if it's a local repository
     repo = _make_absolute(repo)
     cmd = ['hg', 'pull']
     cmd.extend(common_args(**kwargs))
+
     cmd.append(repo)
     run_cmd(cmd, cwd=dest)
 
     if update_dest:
         branch = None
         if 'branch' in kwargs and kwargs['branch']:
             branch = kwargs['branch']
         revision = None
@@ -201,27 +270,38 @@ def push(src, remote, push_new_branches=
     cmd = ['hg', 'push']
     cmd.extend(common_args(**kwargs))
     if push_new_branches:
         cmd.append('--new-branch')
     cmd.append(remote)
     run_cmd(cmd, cwd=src)
 
 def mercurial(repo, dest, branch=None, revision=None, update_dest=True,
-              shareBase=DefaultShareBase, allowUnsharedLocalClones=False):
+              shareBase=DefaultShareBase, allowUnsharedLocalClones=False,
+              clone_by_rev=False, mirrors=None, bundles=None):
     """Makes sure that `dest` is has `revision` or `branch` checked out from
     `repo`.
 
     Do what it takes to make that happen, including possibly clobbering
     dest.
 
     If allowUnsharedLocalClones is True and we're trying to use the share
     extension but fail, then we will be able to clone from the shared repo to
     our destination.  If this is False, the default, then if we don't have the
     share extension we will just clone from the remote repository.
+
+    If `clone_by_rev` is True, use 'hg clone -r <rev>' instead of 'hg clone'.
+    This is slower, but useful when cloning repos with lots of heads.
+
+    If `mirrors` is set, will try and use the mirrors before `repo`.
+
+    If `bundles` is set, will try and download the bundle first and
+    unbundle it instead of doing a full clone. If successful, will pull in
+    new revisions from mirrors or the master repo. If unbundling fails, will
+    fall back to doing a regular clone from mirrors or the master repo.
     """
     dest = os.path.abspath(dest)
     if shareBase is DefaultShareBase:
         shareBase = os.environ.get("HG_SHARE_BASE_DIR", None)
 
     if shareBase:
         # Check that 'hg share' works
         try:
@@ -235,23 +315,39 @@ def mercurial(repo, dest, branch=None, r
                 # Share extension is disabled
                 log.info("Disabling sharing since share extension doesn't seem to work (2)")
                 shareBase = None
         except subprocess.CalledProcessError:
             # The command failed, so disable sharing
             log.info("Disabling sharing since share extension doesn't seem to work (3)")
             shareBase = None
 
+    # Check that our default path is correct
+    if os.path.exists(os.path.join(dest, '.hg')):
+        hgpath = path(dest, "default")
+
+        # Make sure that our default path is correct
+        if hgpath != _make_absolute(repo):
+            log.info("hg path isn't correct (%s should be %s); clobbering", hgpath, _make_absolute(repo))
+            # we need to clobber both the shared checkout and the dest,
+            # since hgrc needs to be in both places
+            remove_path(dest)
+
     # If the working directory already exists and isn't using share we update
     # the working directory directly from the repo, ignoring the sharing
     # settings
     if os.path.exists(dest):
-        if not os.path.exists(os.path.join(dest, ".hg", "sharedpath")):
+        if not os.path.exists(os.path.join(dest, ".hg")):
+            log.warning("%s doesn't appear to be a valid hg directory; clobbering", dest)
+            remove_path(dest)
+        elif not os.path.exists(os.path.join(dest, ".hg", "sharedpath")):
             try:
-                return pull(repo, dest, update_dest=update_dest, branch=branch, revision=revision)
+                return pull(repo, dest, update_dest=update_dest, branch=branch,
+                        revision=revision,
+                        mirrors=mirrors)
             except subprocess.CalledProcessError:
                 log.warning("Error pulling changes into %s from %s; clobbering", dest, repo)
                 log.debug("Exception:", exc_info=True)
                 remove_path(dest)
 
     # If that fails for any reason, and sharing is requested, we'll try to
     # update the shared repository, and then update the working directory from
     # that.
@@ -278,17 +374,18 @@ def mercurial(repo, dest, branch=None, r
                 # Clobber!
                 log.info("We're currently shared from %s, but are being requested to pull from %s (%s); clobbering", dest_sharedPath_data, repo, norm_sharedRepo)
                 remove_path(dest)
 
 
         try:
             log.info("Updating shared repo")
             mercurial(repo, sharedRepo, branch=branch, revision=revision,
-                update_dest=False, shareBase=None)
+                update_dest=False, shareBase=None, clone_by_rev=clone_by_rev,
+                mirrors=mirrors, bundles=bundles)
             if os.path.exists(dest):
                 return update(dest, branch=branch, revision=revision)
 
             try:
                 log.info("Trying to share %s to %s", sharedRepo, dest)
                 return share(sharedRepo, dest, branch=branch, revision=revision)
             except subprocess.CalledProcessError:
                 if not allowUnsharedLocalClones:
@@ -298,27 +395,32 @@ def mercurial(repo, dest, branch=None, r
 
                 log.warning("Error calling hg share from %s to %s;"
                             "falling back to normal clone from shared repo",
                             sharedRepo, dest)
                 # Do a full local clone first, and then update to the
                 # revision we want
                 # This lets us use hardlinks for the local clone if the OS
                 # supports it
-                clone(sharedRepo, dest, update_dest=False)
+                clone(sharedRepo, dest, update_dest=False,
+                        mirrors=mirrors, bundles=bundles)
                 return update(dest, branch=branch, revision=revision)
         except subprocess.CalledProcessError:
             log.warning("Error updating %s from sharedRepo (%s): ", dest, sharedRepo)
             log.debug("Exception:", exc_info=True)
             remove_path(dest)
+    # end if shareBase
 
     if not os.path.exists(os.path.dirname(dest)):
         os.makedirs(os.path.dirname(dest))
+
     # Share isn't available or has failed, clone directly from the source
-    return clone(repo, dest, branch, revision, update_dest=update_dest)
+    return clone(repo, dest, branch, revision,
+            update_dest=update_dest, mirrors=mirrors,
+            bundles=bundles, clone_by_rev=clone_by_rev)
 
 def apply_and_push(localrepo, remote, changer, max_attempts=10,
                    ssh_username=None, ssh_key=None):
     """This function calls `changer' to make changes to the repo, and tries
        its hardest to get them to the origin repo. `changer' must be a
        callable object that receives two arguments: the directory of the local
        repository, and the attempt number. This function will push ALL
        changesets missing from remote."""
@@ -369,8 +471,41 @@ def cleanOutgoingRevs(reponame, remote, 
         run_cmd(['hg', 'strip', '-n', r[REVISION]], cwd=reponame)
 
 def path(src, name='default'):
     """Returns the remote path associated with "name" """
     try:
         return get_output(['hg', 'path', name], cwd=src).strip()
     except subprocess.CalledProcessError:
         return None
+
+def init(dest):
+    """Initializes an empty repo in `dest`"""
+    run_cmd(['hg', 'init', dest])
+
+def unbundle(bundle, dest):
+    """Unbundles the bundle located at `bundle` into `dest`.
+
+    `bundle` can be a local file or remote url."""
+    run_cmd(['hg', 'unbundle', bundle], cwd=dest)
+
+def adjust_paths(dest, **paths):
+    """Adjusts paths in `dest`/.hg/hgrc so that names in `paths` are set to
+    paths[name].
+
+    Note that any comments in the hgrc will be lost if changes are made to the
+    file."""
+    hgrc = os.path.join(dest, '.hg', 'hgrc')
+    config = RawConfigParser()
+    config.read(hgrc)
+
+    if not config.has_section('paths'):
+        config.add_section('paths')
+
+    changed = False
+    for path_name, path_value in paths.items():
+        if (not config.has_option('paths', path_name) or
+                config.get('paths', path_name) != path_value):
+            changed = True
+            config.set('paths', path_name, path_value)
+
+    if changed:
+        config.write(open(hgrc, 'w'))