Bug 1389715 - Vendor latest robustcheckout. r=ted, a=release
authorGregory Szorc <gps@mozilla.com>
Fri, 11 Aug 2017 14:55:24 -0700
changeset 423624 591a279ca4c4c3889318a661d533c6de591ac86f
parent 423623 925fb6cf84e7243a387643ec1ef8fa8b8292e65e
child 423625 57f261c06f4f26deabcb4374f35fa17b7c594267
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted, release
bugs1389715, 1359959, 1361182, 1354824, 1297153
milestone56.0
Bug 1389715 - Vendor latest robustcheckout. r=ted, a=release From revision 19db5f4b5b10f639d24e69a4f21e4e80c6e5bbdd of version-control-tools. Previous revision was 249a47720ddcf896a9f07600c429a1b4492b805e. Changes include: * Use new vfs APIs when available (bug 1359959) * Mark as compatible with Mercurial 4.2 (bug 1361182) * Retry after SSLError (bug 1354824) * Mark as compatible with Mercurial 4.3 * Detect and recover from open locks (bug 1297153) The most significant is the last one. TaskCluster can SIGKILL `hg`, leading to orphaned transactions and locks. This can lead to timeouts or use of a corrupt repository in some scenarios. Those problems should no longer occur. MozReview-Commit-ID: QAOSLyc0xD
testing/mozharness/external_tools/robustcheckout.py
--- a/testing/mozharness/external_tools/robustcheckout.py
+++ b/testing/mozharness/external_tools/robustcheckout.py
@@ -13,37 +13,54 @@ from __future__ import absolute_import
 
 import contextlib
 import errno
 import functools
 import os
 import random
 import re
 import socket
+import ssl
 import time
 import urllib2
 
 from mercurial.i18n import _
 from mercurial.node import hex
 from mercurial import (
     commands,
     error,
     exchange,
     extensions,
     cmdutil,
     hg,
+    registrar,
     scmutil,
     util,
 )
 
-testedwith = '3.7 3.8 3.9 4.0 4.1'
+testedwith = '3.7 3.8 3.9 4.0 4.1 4.2 4.3'
 minimumhgversion = '3.7'
 
 cmdtable = {}
-command = cmdutil.command(cmdtable)
+
+# Mercurial 4.3 introduced registrar.command as a replacement for
+# cmdutil.command.
+if util.safehasattr(registrar, 'command'):
+    command = registrar.command(cmdtable)
+else:
+    command = cmdutil.command(cmdtable)
+
+# Mercurial 4.2 introduced the vfs module and deprecated the symbol in
+# scmutil.
+def getvfs():
+    try:
+        from mercurial.vfs import vfs
+        return vfs
+    except ImportError:
+        return scmutil.vfs
 
 
 if os.name == 'nt':
     import ctypes
 
     # Get a reference to the DeleteFileW function
     # DeleteFileW accepts filenames encoded as a null terminated sequence of
     # wide chars (UTF-16). Python's ctypes.c_wchar_p correctly encodes unicode
@@ -171,50 +188,73 @@ def _docheckout(ui, url, dest, upstream,
 
     def callself():
         return _docheckout(ui, url, dest, upstream, revision, branch, purge,
                            sharebase, networkattemptlimit, networkattempts)
 
     ui.write('ensuring %s@%s is available at %s\n' % (url, revision or branch,
                                                       dest))
 
-    destvfs = scmutil.vfs(dest, audit=False, realpath=True)
+    # We assume that we're the only process on the machine touching the
+    # repository paths that we were told to use. This means our recovery
+    # scenario when things aren't "right" is to just nuke things and start
+    # from scratch. This is easier to implement than verifying the state
+    # of the data and attempting recovery. And in some scenarios (such as
+    # potential repo corruption), it is probably faster, since verifying
+    # repos can take a while.
+
+    destvfs = getvfs()(dest, audit=False, realpath=True)
+
+    def deletesharedstore(path=None):
+        storepath = path or destvfs.read('.hg/sharedpath').strip()
+        if storepath.endswith('.hg'):
+            storepath = os.path.dirname(storepath)
+
+        storevfs = getvfs()(storepath, audit=False)
+        storevfs.rmtree(forcibly=True)
 
     if destvfs.exists() and not destvfs.exists('.hg'):
         raise error.Abort('destination exists but no .hg directory')
 
     # Require checkouts to be tied to shared storage because efficiency.
     if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
         ui.warn('(destination is not shared; deleting)\n')
         destvfs.rmtree(forcibly=True)
 
     # Verify the shared path exists and is using modern pooled storage.
     if destvfs.exists('.hg/sharedpath'):
         storepath = destvfs.read('.hg/sharedpath').strip()
 
         ui.write('(existing repository shared store: %s)\n' % storepath)
 
         if not os.path.exists(storepath):
-            ui.warn('(shared store does not exist; deleting)\n')
+            ui.warn('(shared store does not exist; deleting destination)\n')
             destvfs.rmtree(forcibly=True)
         elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
             ui.warn('(shared store does not belong to pooled storage; '
-                    'deleting to improve efficiency)\n')
+                    'deleting destination to improve efficiency)\n')
             destvfs.rmtree(forcibly=True)
 
+        storevfs = getvfs()(storepath, audit=False)
+        if storevfs.isfileorlink('store/lock'):
+            ui.warn('(shared store has an active lock; assuming it is left '
+                    'over from a previous process and that the store is '
+                    'corrupt; deleting store and destination just to be '
+                    'sure)\n')
+            destvfs.rmtree(forcibly=True)
+            deletesharedstore(storepath)
+
         # FUTURE when we require generaldelta, this is where we can check
         # for that.
 
-    def deletesharedstore():
-        storepath = destvfs.read('.hg/sharedpath').strip()
-        if storepath.endswith('.hg'):
-            storepath = os.path.dirname(storepath)
-
-        storevfs = scmutil.vfs(storepath, audit=False)
-        storevfs.rmtree(forcibly=True)
+    if destvfs.isfileorlink('.hg/wlock'):
+        ui.warn('(dest has an active working directory lock; assuming it is '
+                'left over from a previous process and that the destination '
+                'is corrupt; deleting it just to be sure)\n')
+        destvfs.rmtree(forcibly=True)
 
     def handlerepoerror(e):
         if e.message == _('abandoned transaction found'):
             ui.warn('(abandoned transaction found; trying to recover)\n')
             repo = hg.repository(ui, dest)
             if not repo.recover():
                 ui.warn('(could not recover repo state; '
                         'deleting shared store)\n')
@@ -262,16 +302,23 @@ def _docheckout(ui, url, dest, upstream,
                 ui.warn('(repository is unrelated; deleting)\n')
                 destvfs.rmtree(forcibly=True)
                 return True
             elif e.args[0].startswith(_('stream ended unexpectedly')):
                 ui.warn('%s\n' % e.args[0])
                 # Will raise if failure limit reached.
                 handlenetworkfailure()
                 return True
+        elif isinstance(e, ssl.SSLError):
+            # Assume all SSL errors are due to the network, as Mercurial
+            # should convert non-transport errors like cert validation failures
+            # to error.Abort.
+            ui.warn('ssl error: %s\n' % e)
+            handlenetworkfailure()
+            return True
         elif isinstance(e, urllib2.URLError):
             if isinstance(e.reason, socket.error):
                 ui.warn('socket error: %s\n' % e.reason)
                 handlenetworkfailure()
                 return True
 
         return False
 
@@ -290,17 +337,17 @@ def _docheckout(ui, url, dest, upstream,
 
         if upstream:
             ui.write('(cloning from upstream repo %s)\n' % upstream)
         cloneurl = upstream or url
 
         try:
             res = hg.clone(ui, {}, cloneurl, dest=dest, update=False,
                            shareopts={'pool': sharebase, 'mode': 'identity'})
-        except (error.Abort, urllib2.URLError) as e:
+        except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
             if handlepullerror(e):
                 return callself()
             raise
         except error.RepoError as e:
             return handlerepoerror(e)
         except error.RevlogError as e:
             ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
             deletesharedstore()
@@ -349,17 +396,17 @@ def _docheckout(ui, url, dest, upstream,
                         (branch, checkoutrevision))
 
             if checkoutrevision in repo:
                 ui.warn('(revision already present locally; not pulling)\n')
             else:
                 pullop = exchange.pull(repo, remote, heads=pullrevs)
                 if not pullop.rheads:
                     raise error.Abort('unable to pull requested revision')
-        except (error.Abort, urllib2.URLError) as e:
+        except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
             if handlepullerror(e):
                 return callself()
             raise
         except error.RepoError as e:
             return handlerepoerror(e)
         except error.RevlogError as e:
             ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
             deletesharedstore()