Bug 1392700 - Vendor robustcheckout with capabilities detection. r=glob, a=test-only
authorGregory Szorc <gps@mozilla.com>
Tue, 22 Aug 2017 08:53:51 -0700
changeset 423729 502bf438bc3b62dca9565edeb6896f7324a9ca75
parent 423728 dd7757fd49ffb0d1f987dda5438465f7d2e58d18
child 423730 df69fdfe00c8621ded6704361095a4eb9b185fc5
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglob, test-only
bugs1392700
milestone56.0
Bug 1392700 - Vendor robustcheckout with capabilities detection. r=glob, a=test-only The robustcheckout extension from revision 134574b64ddfa4d7c31977d792761cceca67665a of the version-control-tools repo is vendored without modifications. Changes since last time include printing of the Mercurial version and more robust handling repositories not using modern storage requirements. This patch was not explicitly reviewed by glob. But glob reviewed all the robustcheckout changes since the last vendor. So by the transitive property of code review... MozReview-Commit-ID: CejuVVGpaEy
testing/mozharness/external_tools/robustcheckout.py
--- a/testing/mozharness/external_tools/robustcheckout.py
+++ b/testing/mozharness/external_tools/robustcheckout.py
@@ -18,17 +18,17 @@ import os
 import random
 import re
 import socket
 import ssl
 import time
 import urllib2
 
 from mercurial.i18n import _
-from mercurial.node import hex
+from mercurial.node import hex, nullid
 from mercurial import (
     commands,
     error,
     exchange,
     extensions,
     cmdutil,
     hg,
     registrar,
@@ -157,16 +157,18 @@ def robustcheckout(ui, url, dest, upstre
             raise error.Abort('--revision must be a SHA-1 fragment 12-40 '
                               'characters long')
 
     sharebase = sharebase or ui.config('share', 'pool')
     if not sharebase:
         raise error.Abort('share base directory not defined; refusing to operate',
                           hint='define share.pool config option or pass --sharebase')
 
+    ui.warn('(using Mercurial %s)\n' % util.version())
+
     # worker.backgroundclose only makes things faster if running anti-virus,
     # which our automation doesn't. Disable it.
     ui.setconfig('worker', 'backgroundclose', False)
 
     # By default the progress bar starts after 3s and updates every 0.1s. We
     # change this so it shows and updates every 1.0s.
     # We also tell progress to assume a TTY is present so updates are printed
     # even if there is no known TTY.
@@ -228,28 +230,16 @@ def _docheckout(ui, url, dest, upstream,
         if not os.path.exists(storepath):
             ui.warn('(shared store does not exist; deleting destination)\n')
             destvfs.rmtree(forcibly=True)
         elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
             ui.warn('(shared store does not belong to pooled storage; '
                     'deleting destination to improve efficiency)\n')
             destvfs.rmtree(forcibly=True)
 
-        storevfs = getvfs()(storepath, audit=False)
-        if storevfs.isfileorlink('store/lock'):
-            ui.warn('(shared store has an active lock; assuming it is left '
-                    'over from a previous process and that the store is '
-                    'corrupt; deleting store and destination just to be '
-                    'sure)\n')
-            destvfs.rmtree(forcibly=True)
-            deletesharedstore(storepath)
-
-        # FUTURE when we require generaldelta, this is where we can check
-        # for that.
-
     if destvfs.isfileorlink('.hg/wlock'):
         ui.warn('(dest has an active working directory lock; assuming it is '
                 'left over from a previous process and that the destination '
                 'is corrupt; deleting it just to be sure)\n')
         destvfs.rmtree(forcibly=True)
 
     def handlerepoerror(e):
         if e.message == _('abandoned transaction found'):
@@ -317,35 +307,88 @@ def _docheckout(ui, url, dest, upstream,
         elif isinstance(e, urllib2.URLError):
             if isinstance(e.reason, socket.error):
                 ui.warn('socket error: %s\n' % e.reason)
                 handlenetworkfailure()
                 return True
 
         return False
 
+    # Perform sanity checking of store. We may or may not know the path to the
+    # local store. It depends if we have an existing destvfs pointing to a
+    # share. To ensure we always find a local store, perform the same logic
+    # that Mercurial's pooled storage does to resolve the local store path.
+    cloneurl = upstream or url
+
+    try:
+        clonepeer = hg.peer(ui, {}, cloneurl)
+        rootnode = clonepeer.lookup('0')
+    except error.RepoLookupError:
+        raise error.Abort('unable to resolve root revision from clone '
+                          'source')
+    except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
+        if handlepullerror(e):
+            return callself()
+        raise
+
+    if rootnode == nullid:
+        raise error.Abort('source repo appears to be empty')
+
+    storepath = os.path.join(sharebase, hex(rootnode))
+    storevfs = getvfs()(storepath, audit=False)
+
+    if storevfs.isfileorlink('.hg/store/lock'):
+        ui.warn('(shared store has an active lock; assuming it is left '
+                'over from a previous process and that the store is '
+                'corrupt; deleting store and destination just to be '
+                'sure)\n')
+        if destvfs.exists():
+            destvfs.rmtree(forcibly=True)
+        storevfs.rmtree(forcibly=True)
+
+    if storevfs.exists() and not storevfs.exists('.hg/requires'):
+        ui.warn('(shared store missing requires file; this is a really '
+                'odd failure; deleting store and destination)\n')
+        if destvfs.exists():
+            destvfs.rmtree(forcibly=True)
+        storevfs.rmtree(forcibly=True)
+
+    if storevfs.exists('.hg/requires'):
+        requires = set(storevfs.read('.hg/requires').splitlines())
+        # FUTURE when we require generaldelta, this is where we can check
+        # for that.
+        required = {'dotencode', 'fncache'}
+
+        missing = required - requires
+        if missing:
+            ui.warn('(shared store missing requirements: %s; deleting '
+                    'store and destination to ensure optimal behavior)\n' %
+                    ', '.join(sorted(missing)))
+            if destvfs.exists():
+                destvfs.rmtree(forcibly=True)
+            storevfs.rmtree(forcibly=True)
+
     created = False
 
     if not destvfs.exists():
         # Ensure parent directories of destination exist.
         # Mercurial 3.8 removed ensuredirs and made makedirs race safe.
         if util.safehasattr(util, 'ensuredirs'):
             makedirs = util.ensuredirs
         else:
             makedirs = util.makedirs
 
         makedirs(os.path.dirname(destvfs.base), notindexed=True)
         makedirs(sharebase, notindexed=True)
 
         if upstream:
             ui.write('(cloning from upstream repo %s)\n' % upstream)
-        cloneurl = upstream or url
 
         try:
-            res = hg.clone(ui, {}, cloneurl, dest=dest, update=False,
+            res = hg.clone(ui, {}, clonepeer, dest=dest, update=False,
                            shareopts={'pool': sharebase, 'mode': 'identity'})
         except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
             if handlepullerror(e):
                 return callself()
             raise
         except error.RepoError as e:
             return handlerepoerror(e)
         except error.RevlogError as e: