Bug 1270317 - Add the purgelong Mercurial extension; r=jlund
authorGregory Szorc <gps@mozilla.com>
Fri, 06 May 2016 10:49:25 -0700
changeset 296437 b01744f2d97d0d3c6bb5a53dc58c789a8baf00b0
parent 296436 b893641e2d9fb4f09b926ed22548426b40c2b42b
child 296438 e679c2e0b1b5d8e6ec19deaa44dbbdf9bcf31383
push id76329
push usergszorc@mozilla.com
push dateFri, 06 May 2016 18:16:07 +0000
treeherdermozilla-inbound@273d52aff2ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlund
bugs1270317
milestone49.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1270317 - Add the purgelong Mercurial extension; r=jlund The build/tools repo has a "purgelong" extension that is used to delete long filenames on Windows. Without this extension, the APIs Mercurial uses may run into path length issues and `hg purge` will fail. This commit is a straight import of the purgelong extension from https://hg.mozilla.org/build/tools minus the shebang line, which isn't needed. MozReview-Commit-ID: FIrEeWDf2Dl
testing/mozharness/external_tools/purgelong.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/external_tools/purgelong.py
@@ -0,0 +1,117 @@
+'''
+Mercurial extension to enable purging long filenames on Windows
+
+It's possible to have filenaems that exceed the MAX_PATH limit (256 characters)
+on Windows, rendering them un-purgeable with the regular python API calls.
+
+To work around this limitation, we can use the DeleteFileW API
+(https://msdn.microsoft.com/en-us/library/windows/desktop/aa363915%28v=vs.85%29.aspx)
+and prefix the filename with \\?\.
+
+This extension needs to monkeypatch other modules in order to function. It
+attempts to be very conservative, and only applies the patches for the
+duration of the purge() command. The original functions are restored after
+the purge() command exits.
+'''
+from contextlib import contextmanager
+from functools import partial
+import os
+import errno
+
+import mercurial.extensions
+import mercurial.util
+
+if os.name == 'nt':
+    import ctypes
+
+    # Get a reference to the DeleteFileW function
+    # DeleteFileW accepts filenames encoded as a null terminated sequence of
+    # wide chars (UTF-16). Python's ctypes.c_wchar_p correctly encodes unicode
+    # strings to null terminated UTF-16 strings.
+    # However, we receive (byte) strings from mercurial. When these are passed
+    # to DeleteFileW via the c_wchar_p type, they are implicitly decoded via
+    # the 'mbcs' encoding on windows.
+    kernel32 = ctypes.windll.kernel32
+    DeleteFile = kernel32.DeleteFileW
+    DeleteFile.argtypes = [ctypes.c_wchar_p]
+    DeleteFile.restype = ctypes.c_bool
+
+    def unlink_long(fn):
+        normalized_path = '\\\\?\\' + os.path.normpath(fn)
+        if not DeleteFile(normalized_path):
+            raise OSError(errno.EPERM, "couldn't remove long path", fn)
+
+# Not needed on other platforms, but is handy for testing
+else:
+    def unlink_long(fn):
+        os.unlink(fn)
+
+
+def unlink_wrapper(unlink_orig, fn, ui):
+    '''Calls the original unlink function, and if that fails, calls
+    unlink_long'''
+    try:
+        ui.debug('calling unlink_orig %s\n' % fn)
+        return unlink_orig(fn)
+    except WindowsError, e:
+        # windows error 3 corresponds to ERROR_PATH_NOT_FOUND
+        # only handle this case; re-raise the exception for other kinds of
+        # failures
+        if e.winerror != 3:
+            raise
+        ui.debug('caught WindowsError ERROR_PATH_NOT_FOUND; '
+                 'calling unlink_long %s\n' % fn)
+        return unlink_long(fn)
+
+
+@contextmanager
+def wrap_unlink(ui):
+    '''Context manager that patches the required functions that are used by
+    the purge extension to remove files. When exiting the context manager
+    the original functions are restored.'''
+    version = mercurial.util.version()
+    if version >= '3.2':
+        # hg 3.2 and higher use util.unlink for purging
+        purgemod = mercurial.extensions.find('purge')
+        to_wrap = [(purgemod.util, 'unlink')]
+    else:
+        # hg 3.1 and earlier use os.remove directly
+        to_wrap = [(os, 'remove')]
+
+    # pass along the ui object to the unlink_wrapper so we can get logging out
+    # of it
+    wrapped = partial(unlink_wrapper, ui=ui)
+
+    # Wrap the original function(s) with our unlink_wrapper
+    originals = {}
+    for mod, func in to_wrap:
+        ui.debug('wrapping %s %s\n' % (mod, func))
+        originals[mod, func] = mercurial.extensions.wrapfunction(
+            mod, func, wrapped)
+
+    try:
+        yield
+    finally:
+        # Restore the originals
+        for mod, func in to_wrap:
+            ui.debug('restoring %s %s\n' % (mod, func))
+            setattr(mod, func, originals[mod, func])
+
+
+def purge_wrapper(orig, ui, *args, **kwargs):
+    '''Runs the original purge() command inside the wrap_unlink() context
+    manager.'''
+    with wrap_unlink(ui):
+        return orig(ui, *args, **kwargs)
+
+
+def extsetup(ui):
+    try:
+        purgemod = mercurial.extensions.find('purge')
+    except KeyError:
+        ui.warn('purge extension not found; '
+                'not enabling long filename support\n')
+        return
+
+    ui.note('enabling long filename support for purge\n')
+    mercurial.extensions.wrapcommand(purgemod.cmdtable, 'purge', purge_wrapper)