Bug 1512075 - Upgrade to latest robustcheckout; r=ted
authorGregory Szorc <gps@mozilla.com>
Tue, 04 Dec 2018 23:30:29 +0000
changeset 449343 c28a868a45f5b368f687f2f6e9b32914a9d560c0
parent 449342 7dc99db92ea4dee6bd7a02b20067fd3b64e1f352
child 449344 6397590938811dfaec66b509a9f167c454a6779f
push id35158
push usercsabou@mozilla.com
push dateWed, 05 Dec 2018 10:19:05 +0000
treeherdermozilla-central@f1f136ea674c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1512075
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1512075 - Upgrade to latest robustcheckout; r=ted From changeset d0828787e64fa55b535c7e783bc97612f5c30cff from version-control-tools repository. Differential Revision: https://phabricator.services.mozilla.com/D13767
testing/mozharness/external_tools/robustcheckout.py
--- a/testing/mozharness/external_tools/robustcheckout.py
+++ b/testing/mozharness/external_tools/robustcheckout.py
@@ -161,16 +161,25 @@ def wrapunlink(ui):
 
 
 def purgewrapper(orig, ui, *args, **kwargs):
     '''Runs original purge() command with unlink monkeypatched.'''
     with wrapunlink(ui):
         return orig(ui, *args, **kwargs)
 
 
+def peerlookup(remote, v):
+    # TRACKING hg46 4.6 added commandexecutor API.
+    if util.safehasattr(remote, 'commandexecutor'):
+        with remote.commandexecutor() as e:
+            return e.callcommand('lookup', {'key': v}).result()
+    else:
+        return remote.lookup(v)
+
+
 @command('robustcheckout', [
     ('', 'upstream', '', 'URL of upstream repo to clone from'),
     ('r', 'revision', '', 'Revision to check out'),
     ('b', 'branch', '', 'Branch to check out'),
     ('', 'purge', False, 'Whether to purge the working directory'),
     ('', 'sharebase', '', 'Directory where shared repos should be placed'),
     ('', 'networkattempts', 3, 'Maximum number of attempts for network '
                                'operations'),
@@ -259,26 +268,78 @@ def robustcheckout(ui, url, dest, upstre
     # otherwise we're at the whim of whatever configs are used in automation.
     ui.setconfig('progress', 'delay', 1.0)
     ui.setconfig('progress', 'refresh', 1.0)
     ui.setconfig('progress', 'assume-tty', True)
 
     sharebase = os.path.realpath(sharebase)
 
     optimes = []
+    behaviors = set()
     start = time.time()
 
     try:
         return _docheckout(ui, url, dest, upstream, revision, branch, purge,
-                           sharebase, optimes, networkattempts,
+                           sharebase, optimes, behaviors, networkattempts,
                            sparse_profile=sparseprofile)
     finally:
         overall = time.time() - start
+
+        # We store the overall time multiple ways in order to help differentiate
+        # the various "flavors" of operations.
+
+        # ``overall`` is always the total operation time.
         optimes.append(('overall', overall))
 
+        def record_op(name):
+            # If special behaviors due to "corrupt" storage occur, we vary the
+            # name to convey that.
+            if 'remove-store' in behaviors:
+                name += '_rmstore'
+            if 'remove-wdir' in behaviors:
+                name += '_rmwdir'
+
+            optimes.append((name, overall))
+
+        # We break out overall operations primarily by their network interaction
+        # We have variants within for working directory operations.
+        if 'clone' in behaviors:
+            record_op('overall_clone')
+
+            if 'sparse-update' in behaviors:
+                record_op('overall_clone_sparsecheckout')
+            else:
+                record_op('overall_clone_fullcheckout')
+
+        elif 'pull' in behaviors:
+            record_op('overall_pull')
+
+            if 'sparse-update' in behaviors:
+                record_op('overall_pull_sparsecheckout')
+            else:
+                record_op('overall_pull_fullcheckout')
+
+            if 'empty-wdir' in behaviors:
+                record_op('overall_pull_emptywdir')
+            else:
+                record_op('overall_pull_populatedwdir')
+
+        else:
+            record_op('overall_nopull')
+
+            if 'sparse-update' in behaviors:
+                record_op('overall_nopull_sparsecheckout')
+            else:
+                record_op('overall_nopull_fullcheckout')
+
+            if 'empty-wdir' in behaviors:
+                record_op('overall_nopull_emptywdir')
+            else:
+                record_op('overall_nopull_populatedwdir')
+
         if 'TASKCLUSTER_INSTANCE_TYPE' in os.environ:
             perfherder = {
                 'framework': {
                     'name': 'vcs',
                 },
                 'suites': [],
             }
             for op, duration in optimes:
@@ -290,29 +351,30 @@ def robustcheckout(ui, url, dest, upstre
                     'extraOptions': [os.environ['TASKCLUSTER_INSTANCE_TYPE']],
                     'subtests': [],
                 })
 
             ui.write('PERFHERDER_DATA: %s\n' % json.dumps(perfherder,
                                                           sort_keys=True))
 
 def _docheckout(ui, url, dest, upstream, revision, branch, purge, sharebase,
-                optimes, networkattemptlimit, networkattempts=None,
+                optimes, behaviors, networkattemptlimit, networkattempts=None,
                 sparse_profile=None):
     if not networkattempts:
         networkattempts = [1]
 
     def callself():
         return _docheckout(ui, url, dest, upstream, revision, branch, purge,
-                           sharebase, optimes, networkattemptlimit,
+                           sharebase, optimes, behaviors, networkattemptlimit,
                            networkattempts=networkattempts,
                            sparse_profile=sparse_profile)
 
     @contextlib.contextmanager
-    def timeit(op):
+    def timeit(op, behavior):
+        behaviors.add(behavior)
         errored = False
         try:
             start = time.time()
             yield
         except Exception:
             errored = True
             raise
         finally:
@@ -359,50 +421,50 @@ def _docheckout(ui, url, dest, upstream,
     if not sparse_profile and destvfs.exists('.hg/sparse'):
         raise error.Abort('cannot use non-sparse checkout on existing sparse '
                           'checkout',
                           hint='use a separate working directory to use sparse')
 
     # Require checkouts to be tied to shared storage because efficiency.
     if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
         ui.warn('(destination is not shared; deleting)\n')
-        with timeit('remove_unshared_dest'):
+        with timeit('remove_unshared_dest', 'remove-wdir'):
             destvfs.rmtree(forcibly=True)
 
     # Verify the shared path exists and is using modern pooled storage.
     if destvfs.exists('.hg/sharedpath'):
         storepath = destvfs.read('.hg/sharedpath').strip()
 
         ui.write('(existing repository shared store: %s)\n' % storepath)
 
         if not os.path.exists(storepath):
             ui.warn('(shared store does not exist; deleting destination)\n')
-            with timeit('removed_missing_shared_store'):
+            with timeit('removed_missing_shared_store', 'remove-wdir'):
                 destvfs.rmtree(forcibly=True)
         elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
             ui.warn('(shared store does not belong to pooled storage; '
                     'deleting destination to improve efficiency)\n')
-            with timeit('remove_unpooled_store'):
+            with timeit('remove_unpooled_store', 'remove-wdir'):
                 destvfs.rmtree(forcibly=True)
 
     if destvfs.isfileorlink('.hg/wlock'):
         ui.warn('(dest has an active working directory lock; assuming it is '
                 'left over from a previous process and that the destination '
                 'is corrupt; deleting it just to be sure)\n')
-        with timeit('remove_locked_wdir'):
+        with timeit('remove_locked_wdir', 'remove-wdir'):
             destvfs.rmtree(forcibly=True)
 
     def handlerepoerror(e):
         if e.message == _('abandoned transaction found'):
             ui.warn('(abandoned transaction found; trying to recover)\n')
             repo = hg.repository(ui, dest)
             if not repo.recover():
                 ui.warn('(could not recover repo state; '
                         'deleting shared store)\n')
-                with timeit('remove_unrecovered_shared_store'):
+                with timeit('remove_unrecovered_shared_store', 'remove-store'):
                     deletesharedstore()
 
             ui.warn('(attempting checkout from beginning)\n')
             return callself()
 
         raise
 
     # At this point we either have an existing working directory using
@@ -471,17 +533,17 @@ def _docheckout(ui, url, dest, upstream,
     # Perform sanity checking of store. We may or may not know the path to the
     # local store. It depends if we have an existing destvfs pointing to a
     # share. To ensure we always find a local store, perform the same logic
     # that Mercurial's pooled storage does to resolve the local store path.
     cloneurl = upstream or url
 
     try:
         clonepeer = hg.peer(ui, {}, cloneurl)
-        rootnode = clonepeer.lookup('0')
+        rootnode = peerlookup(clonepeer, '0')
     except error.RepoLookupError:
         raise error.Abort('unable to resolve root revision from clone '
                           'source')
     except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
         if handlepullerror(e):
             return callself()
         raise
 
@@ -492,48 +554,48 @@ def _docheckout(ui, url, dest, upstream,
     storevfs = getvfs()(storepath, audit=False)
 
     if storevfs.isfileorlink('.hg/store/lock'):
         ui.warn('(shared store has an active lock; assuming it is left '
                 'over from a previous process and that the store is '
                 'corrupt; deleting store and destination just to be '
                 'sure)\n')
         if destvfs.exists():
-            with timeit('remove_dest_active_lock'):
+            with timeit('remove_dest_active_lock', 'remove-wdir'):
                 destvfs.rmtree(forcibly=True)
 
-        with timeit('remove_shared_store_active_lock'):
+        with timeit('remove_shared_store_active_lock', 'remove-store'):
             storevfs.rmtree(forcibly=True)
 
     if storevfs.exists() and not storevfs.exists('.hg/requires'):
         ui.warn('(shared store missing requires file; this is a really '
                 'odd failure; deleting store and destination)\n')
         if destvfs.exists():
-            with timeit('remove_dest_no_requires'):
+            with timeit('remove_dest_no_requires', 'remove-wdir'):
                 destvfs.rmtree(forcibly=True)
 
-        with timeit('remove_shared_store_no_requires'):
+        with timeit('remove_shared_store_no_requires', 'remove-store'):
             storevfs.rmtree(forcibly=True)
 
     if storevfs.exists('.hg/requires'):
         requires = set(storevfs.read('.hg/requires').splitlines())
         # FUTURE when we require generaldelta, this is where we can check
         # for that.
         required = {'dotencode', 'fncache'}
 
         missing = required - requires
         if missing:
             ui.warn('(shared store missing requirements: %s; deleting '
                     'store and destination to ensure optimal behavior)\n' %
                     ', '.join(sorted(missing)))
             if destvfs.exists():
-                with timeit('remove_dest_missing_requires'):
+                with timeit('remove_dest_missing_requires', 'remove-wdir'):
                     destvfs.rmtree(forcibly=True)
 
-            with timeit('remove_shared_store_missing_requires'):
+            with timeit('remove_shared_store_missing_requires', 'remove-store'):
                 storevfs.rmtree(forcibly=True)
 
     created = False
 
     if not destvfs.exists():
         # Ensure parent directories of destination exist.
         # Mercurial 3.8 removed ensuredirs and made makedirs race safe.
         if util.safehasattr(util, 'ensuredirs'):
@@ -543,29 +605,29 @@ def _docheckout(ui, url, dest, upstream,
 
         makedirs(os.path.dirname(destvfs.base), notindexed=True)
         makedirs(sharebase, notindexed=True)
 
         if upstream:
             ui.write('(cloning from upstream repo %s)\n' % upstream)
 
         try:
-            with timeit('clone'):
+            with timeit('clone', 'clone'):
                 shareopts = {'pool': sharebase, 'mode': 'identity'}
                 res = hg.clone(ui, {}, clonepeer, dest=dest, update=False,
                                shareopts=shareopts)
         except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
             if handlepullerror(e):
                 return callself()
             raise
         except error.RepoError as e:
             return handlerepoerror(e)
         except error.RevlogError as e:
             ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
-            with timeit('remove_shared_store_revlogerror'):
+            with timeit('remove_shared_store_revlogerror', 'remote-store'):
                 deletesharedstore()
             return callself()
 
         # TODO retry here.
         if res is None:
             raise error.Abort('clone failed')
 
         # Verify it is using shared pool storage.
@@ -599,27 +661,27 @@ def _docheckout(ui, url, dest, upstream,
             havewantedrev = True
 
     if not havewantedrev:
         ui.write('(pulling to obtain %s)\n' % (revision or branch,))
 
         remote = None
         try:
             remote = hg.peer(repo, {}, url)
-            pullrevs = [remote.lookup(revision or branch)]
+            pullrevs = [peerlookup(remote, revision or branch)]
             checkoutrevision = hex(pullrevs[0])
             if branch:
                 ui.warn('(remote resolved %s to %s; '
                         'result is not deterministic)\n' %
                         (branch, checkoutrevision))
 
             if checkoutrevision in repo:
                 ui.warn('(revision already present locally; not pulling)\n')
             else:
-                with timeit('pull'):
+                with timeit('pull', 'pull'):
                     pullop = exchange.pull(repo, remote, heads=pullrevs)
                     if not pullop.rheads:
                         raise error.Abort('unable to pull requested revision')
         except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
             if handlepullerror(e):
                 return callself()
             raise
         except error.RepoError as e:
@@ -645,31 +707,36 @@ def _docheckout(ui, url, dest, upstream,
         # See https://bz.mercurial-scm.org/show_bug.cgi?id=5626. Force
         # purging by monkeypatching the sparse matcher.
         try:
             old_sparse_fn = getattr(repo.dirstate, '_sparsematchfn', None)
             if old_sparse_fn is not None:
                 assert supported_hg(), 'Mercurial version not supported (must be 4.3+)'
                 repo.dirstate._sparsematchfn = lambda: matchmod.always(repo.root, '')
 
-            with timeit('purge'):
+            with timeit('purge', 'purge'):
                 if purgeext.purge(ui, repo, all=True, abort_on_err=True,
                                   # The function expects all arguments to be
                                   # defined.
                                   **{'print': None,
                                      'print0': None,
                                      'dirs': None,
                                      'files': None}):
                     raise error.Abort('error purging')
         finally:
             if old_sparse_fn is not None:
                 repo.dirstate._sparsematchfn = old_sparse_fn
 
     # Update the working directory.
 
+    if repo['.'].node() == nullid:
+        behaviors.add('empty-wdir')
+    else:
+        behaviors.add('populated-wdir')
+
     if sparse_profile:
         sparsemod = getsparse()
 
         # By default, Mercurial will ignore unknown sparse profiles. This could
         # lead to a full checkout. Be more strict.
         try:
             repo.filectx(sparse_profile, changeid=checkoutrevision).data()
         except error.ManifestLookupError:
@@ -695,28 +762,30 @@ def _docheckout(ui, url, dest, upstream,
             else:
                 ui.write('(setting sparse config to profile %s)\n' %
                          sparse_profile)
 
             # If doing an incremental update, this will perform two updates:
             # one to change the sparse profile and another to update to the new
             # revision. This is not desired. But there's not a good API in
             # Mercurial to do this as one operation.
-            with repo.wlock(), timeit('sparse_update_config'):
+            with repo.wlock(), timeit('sparse_update_config',
+                                      'sparse-update-config'):
                 fcounts = map(len, sparsemod._updateconfigandrefreshwdir(
                     repo, [], [], [sparse_profile], force=True))
 
                 repo.ui.status('%d files added, %d files dropped, '
                                '%d files conflicting\n' % tuple(fcounts))
 
             ui.write('(sparse refresh complete)\n')
 
     op = 'update_sparse' if sparse_profile else 'update'
+    behavior = 'update-sparse' if sparse_profile else 'update'
 
-    with timeit(op):
+    with timeit(op, behavior):
         if commands.update(ui, repo, rev=checkoutrevision, clean=True):
             raise error.Abort('error updating')
 
     ui.write('updated to %s\n' % checkoutrevision)
 
     # HACK workaround https://bz.mercurial-scm.org/show_bug.cgi?id=5905
     # and https://bugzilla.mozilla.org/show_bug.cgi?id=1462323.
     #