Bug 851270 - Don't clobber the source checkout. r=catlee FENNEC_21_0b7_BUILD1 FENNEC_21_0b7_RELEASE FIREFOX_21_0b7_BUILD1 FIREFOX_21_0b7_RELEASE
authorMassimo Gervasini <mgervasini@mozilla.com>
Mon, 06 May 2013 16:19:54 +0200
changeset 3667 db7250c6e05ee59c849a0d2bffda23312008c0f3
parent 3666 c16a2d033185f398a864818e7e246b9e01cc4231
child 3668 6c2de063931d7df527eaa837c4b454ca785774b7
push id2651
push usermgervasini@mozilla.com
push dateMon, 06 May 2013 14:26:36 +0000
reviewerscatlee
bugs851270
Bug 851270 - Don't clobber the source checkout. r=catlee
.gitignore
.hgignore
buildfarm/utils/hgtool.py
lib/python/mozilla_buildtools/test/test_util_hg.py
lib/python/util/commands.py
lib/python/util/hg.py
tox.ini
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,11 @@
 release-runner*.ini
 master_config.json
 buildbot-configs
 buildbotcustom
 tools
+.coverage
+.tox
+coverage.xml
+lib/python/buildtools.egg-info/
+nosetests.xml
+
--- a/.hgignore
+++ b/.hgignore
@@ -9,8 +9,14 @@ slavealloc\.db
 \..*\.swp
 twistd\.pid
 .*\.egg-info
 dist/
 slavealloc.log
 release-runner.ini
 lib/python/slavealloc/www/icons
 lib/python/slavealloc/www/js/bugzilla.js
+.coverage
+.tox
+coverage.xml
+lib/python/buildtools.egg-info/
+nosetests.xml
+
--- a/buildfarm/utils/hgtool.py
+++ b/buildfarm/utils/hgtool.py
@@ -43,16 +43,18 @@ if __name__ == '__main__':
         "--clone-by-revision", dest="clone_by_rev", action="store_true",
         help="do initial clone with -r <rev> instead of cloning the entire repo. "
              "This is slower but is useful when cloning repositories with many "
              "heads which may timeout otherwise.")
     parser.add_option("--mirror", dest="mirrors", action="append",
                       help="add a mirror to try cloning/pulling from before repo")
     parser.add_option("--bundle", dest="bundles", action="append",
                       help="add a bundle to try downloading/unbundling from before doing a full clone")
+    parser.add_option("--purge", dest="auto_purge", action="store_true",
+                      help="Purge the destination directory (if it exists).")
 
     options, args = parser.parse_args()
 
     logging.basicConfig(level=options.loglevel, format="%(message)s")
 
     if len(args) not in (1, 2):
         parser.error("Invalid number of arguments")
 
@@ -80,11 +82,12 @@ if __name__ == '__main__':
             remove_path(dest)
         if options.shared_dir and out(options.shared_dir, repo):
             remove_path(options.shared_dir)
 
     got_revision = mercurial(repo, dest, options.branch, options.revision,
                              shareBase=options.shared_dir,
                              clone_by_rev=options.clone_by_rev,
                              mirrors=options.mirrors,
-                             bundles=options.bundles)
+                             bundles=options.bundles,
+                             autoPurge=options.auto_purge)
 
     print "Got revision %s" % got_revision
--- a/lib/python/mozilla_buildtools/test/test_util_hg.py
+++ b/lib/python/mozilla_buildtools/test/test_util_hg.py
@@ -1,17 +1,17 @@
 import unittest
 import tempfile
 import shutil
 import os
 import subprocess
 
 import util.hg as hg
 from util.hg import clone, pull, update, hg_ver, mercurial, _make_absolute, \
-    share, push, apply_and_push, HgUtilError, make_hg_url, get_branch, \
+    share, push, apply_and_push, HgUtilError, make_hg_url, get_branch, purge, \
     get_branches, path, init, unbundle, adjust_paths, is_hg_cset, commit, tag
 from util.commands import run_cmd, get_output
 
 
 def getRevisions(dest):
     retval = []
     for rev in get_output(['hg', 'log', '-R', dest, '--template', '{node|short}\n']).split('\n'):
         rev = rev.strip()
@@ -31,18 +31,18 @@ def getRevInfo(dest, rev):
     }
     if len(output) > 2:
         info['tags'] = output[2].split()
     return info
 
 
 def getTags(dest):
     tags = []
-    for tag in get_output(['hg', 'tags', '-R', dest]).splitlines():
-        tags.append(tag.split()[0])
+    for t in get_output(['hg', 'tags', '-R', dest]).splitlines():
+        tags.append(t.split()[0])
     return tags
 
 
 class TestMakeAbsolute(unittest.TestCase):
     def testAbsolutePath(self):
         self.assertEquals(_make_absolute("/foo/bar"), "/foo/bar")
 
     def testRelativePath(self):
@@ -293,16 +293,87 @@ class TestHg(unittest.TestCase):
         self.assertEquals(getRevisions(self.wc), self.revisions)
 
     def testPushWithRevision(self):
         clone(self.repodir, self.wc, revision=self.revisions[-2],
               clone_by_rev=True)
         push(src=self.repodir, remote=self.wc, revision=self.revisions[-1])
         self.assertEquals(getRevisions(self.wc), self.revisions[-2:])
 
+    def testPurgeUntrackedFile(self):
+        rev = clone(self.repodir, self.wc, update_dest=False)
+        self.assertEquals(rev, None)
+        fileToPurge=os.path.join(self.wc, 'fileToPurge')
+        with file(fileToPurge, 'a') as f:
+            f.write('purgeme')
+        purge(self.wc)
+        self.assertFalse(os.path.exists(fileToPurge))
+
+    def testPurgeUntrackedDirectory(self):
+        rev = clone(self.repodir, self.wc, update_dest=False)
+        self.assertEquals(rev, None)
+        directoryToPurge = os.path.join(self.wc, 'directoryToPurge')
+        os.makedirs(directoryToPurge)
+        purge(self.wc)
+        self.assertFalse(os.path.isdir(directoryToPurge))
+
+    def testPurgeTrackedFile(self):
+        rev = clone(self.repodir, self.wc, update_dest=False)
+        self.assertEquals(rev, None)
+        fileToModify = os.path.join(self.wc, 'purgetest.txt')
+        open(fileToModify, 'w').write('hello!')
+        run_cmd(['hg', 'add', 'purgetest.txt'], cwd=self.wc)
+        run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.wc)
+        with open(fileToModify, 'w') as f:
+            f.write('just a test')
+        purge(self.wc)
+        content = open(fileToModify).read()
+        self.assertEqual(content, 'just a test')
+
+    def testPurgeUntrackedFile(self):
+        clone(self.repodir, self.wc)
+        fileToPurge=os.path.join(self.wc, 'fileToPurge')
+        with file(fileToPurge, 'a') as f:
+            f.write('purgeme')
+        purge(self.wc)
+        self.assertFalse(os.path.exists(fileToPurge))
+
+    def testPurgeUntrackedDirectory(self):
+        clone(self.repodir, self.wc)
+        directoryToPurge = os.path.join(self.wc, 'directoryTopPurge')
+        os.makedirs(directoryToPurge)
+        purge(directoryToPurge)
+        self.assertFalse(os.path.isdir(directoryToPurge))
+
+    def testPurgeVeryLongPath(self):
+        clone(self.repodir, self.wc)
+        # now create a very long path name
+        longPath = self.wc
+        for new_dir in xrange(1,64):
+            longPath = os.path.join(longPath, str(new_dir))
+        os.makedirs(longPath)
+        self.assertTrue(os.path.isdir(longPath))
+        purge(self.wc)
+        self.assertFalse(os.path.isdir(longPath))
+
+    def testPurgeAFreshClone(self):
+        clone(self.repodir, self.wc)
+        purge(self.wc)
+        self.assertTrue(os.path.exists(os.path.join(self.wc, 'hello.txt')))
+
+    def testPurgeTrackedFile(self):
+        clone(self.repodir, self.wc)
+        fileToModify = os.path.join(self.wc, 'hello.txt')
+        with open(fileToModify, 'w') as f:
+            f.write('hello!')
+        purge(self.wc)
+        with open(fileToModify, 'r') as f:
+            content = f.read()
+        self.assertEqual(content, 'hello!')
+
     def testMercurial(self):
         rev = mercurial(self.repodir, self.wc)
         self.assertEquals(rev, self.revisions[0])
 
     def testPushNewBranchesNotAllowed(self):
         clone(self.repodir, self.wc, revision=self.revisions[0],
               clone_by_rev=True)
         self.assertRaises(Exception, push, self.repodir, self.wc,
--- a/lib/python/util/commands.py
+++ b/lib/python/util/commands.py
@@ -1,16 +1,22 @@
 """Functions for running commands"""
 import subprocess
 import os
 import time
-
+import platform
 import logging
 log = logging.getLogger(__name__)
 
+try:
+    import win32file
+    import win32api
+    PYWIN32 = True
+except ImportError:
+    PYWIN32 = False
 
 def log_cmd(cmd, **kwargs):
     # cwd is special in that we always want it printed, even if it's not
     # explicitly chosen
     kwargs = kwargs.copy()
     if 'cwd' not in kwargs:
         kwargs['cwd'] = os.getcwd()
     log.info("command: START")
@@ -156,16 +162,21 @@ def get_output(cmd, include_stderr=False
         log.info("command: END (%.2f elapsed)\n", elapsed)
 
 
 def remove_path(path):
     """This is a replacement for shutil.rmtree that works better under
     windows. Thanks to Bear at the OSAF for the code.
     (Borrowed from buildbot.slave.commands)"""
     log.debug("Removing %s", path)
+
+    if _is_windows():
+        _rmtree_windows(path)
+        return
+
     if not os.path.exists(path):
         # This handles broken links
         if os.path.islink(path):
             os.remove(path)
         return
 
     if os.path.islink(path):
         os.remove(path)
@@ -188,8 +199,56 @@ def remove_path(path):
         if os.path.isdir(full_name):
             remove_path(full_name)
         else:
             # Don't try to chmod links
             if not os.path.islink(full_name):
                 os.chmod(full_name, 0700)
             os.remove(full_name)
     os.rmdir(path)
+
+
+# _is_windows and _rmtree_windows taken
+# from mozharness
+
+def _is_windows():
+    system = platform.system()
+    if system in ("Windows", "Microsoft"):
+        return True
+    if system.startswith("CYGWIN"):
+        return True
+    if os.name == 'nt':
+        return True
+
+def _rmtree_windows(path):
+    """ Windows-specific rmtree that handles path lengths longer than MAX_PATH.
+        Ported from clobberer.py.
+    """
+    log.info("Using _rmtree_windows ...")
+    assert _is_windows()
+    path = os.path.realpath(path)
+    full_path = '\\\\?\\' + path
+    if not os.path.exists(full_path):
+        return
+    if not PYWIN32:
+        if not os.path.isdir(path):
+            return run_cmd('del /F /Q "%s"' % path)
+        else:
+            return run_cmd('rmdir /S /Q "%s"' % path)
+    # Make sure directory is writable
+    win32file.SetFileAttributesW('\\\\?\\' + path, win32file.FILE_ATTRIBUTE_NORMAL)
+    # Since we call rmtree() with a file, sometimes
+    if not os.path.isdir('\\\\?\\' + path):
+        return win32file.DeleteFile('\\\\?\\' + path)
+
+    for ffrec in win32api.FindFiles('\\\\?\\' + path + '\\*.*'):
+        file_attr = ffrec[0]
+        name = ffrec[8]
+        if name == '.' or name == '..':
+            continue
+        full_name = os.path.join(path, name)
+
+        if file_attr & win32file.FILE_ATTRIBUTE_DIRECTORY:
+            _rmtree_windows(full_name)
+        else:
+            win32file.SetFileAttributesW('\\\\?\\' + full_name, win32file.FILE_ATTRIBUTE_NORMAL)
+            win32file.DeleteFile('\\\\?\\' + full_name)
+    win32file.RemoveDirectory('\\\\?\\' + path)
--- a/lib/python/util/hg.py
+++ b/lib/python/util/hg.py
@@ -73,17 +73,17 @@ def get_branches(path):
         branches.append(line.split()[0])
     return branches
 
 
 def is_hg_cset(rev):
     """Retruns True if passed revision represents a valid HG revision
     (long or short(er) 40 bit hex)"""
     try:
-        _ = int(rev, 16)
+        int(rev, 16)
         return True
     except (TypeError, ValueError):
         return False
 
 
 def hg_ver():
     """Returns the current version of hg, as a tuple of
     (major, minor, build)"""
@@ -94,16 +94,24 @@ def hg_ver():
         if len(bits) < 3:
             bits += (0,)
         ver = tuple(int(b) for b in bits)
     else:
         ver = (0, 0, 0)
     log.debug("Running hg version %s", ver)
     return ver
 
+def purge(dest):
+    """Purge the repository of all untracked and ignored files."""
+    try:
+        run_cmd(['hg', '--config', 'extensions.purge=', 'purge', '-a', '--all',
+                  dest], cwd=os.path.normpath(os.path.join(dest, '..')))
+    except subprocess.CalledProcessError as e:
+        log.debug('purge failed: %s' %e)
+        raise
 
 def update(dest, branch=None, revision=None):
     """Updates working copy `dest` to `branch` or `revision`.  If neither is
     set then the working copy will be updated to the latest revision on the
     current branch.  Local changes will be discarded."""
     # If we have a revision, switch to that
     if revision is not None:
         cmd = ['hg', 'update', '-C', '-r', revision]
@@ -182,17 +190,17 @@ def clone(repo, dest, branch=None, revis
                 log.exception("Problem cloning from mirror %s", mirror)
                 continue
         else:
             log.info("Pulling from mirrors failed; falling back to %s", repo)
             # We may have a partial repo here; mercurial() copes with that
             # We need to make sure our paths are correct though
             if os.path.exists(os.path.join(dest, '.hg')):
                 adjust_paths(dest, default=repo)
-            return mercurial(repo, dest, branch, revision,
+            return mercurial(repo, dest, branch, revision, autoPurge=True,
                              update_dest=update_dest, clone_by_rev=clone_by_rev)
 
     cmd = ['hg', 'clone']
     if not update_dest:
         cmd.append('-U')
 
     if clone_by_rev:
         if revision:
@@ -312,17 +320,17 @@ def push(src, remote, push_new_branches=
     if push_new_branches:
         cmd.append('--new-branch')
     cmd.append(remote)
     run_cmd(cmd, cwd=src)
 
 
 def mercurial(repo, dest, branch=None, revision=None, update_dest=True,
               shareBase=DefaultShareBase, allowUnsharedLocalClones=False,
-              clone_by_rev=False, mirrors=None, bundles=None):
+              clone_by_rev=False, mirrors=None, bundles=None, autoPurge=False):
     """Makes sure that `dest` is has `revision` or `branch` checked out from
     `repo`.
 
     Do what it takes to make that happen, including possibly clobbering
     dest.
 
     If allowUnsharedLocalClones is True and we're trying to use the share
     extension but fail, then we will be able to clone from the shared repo to
@@ -379,16 +387,18 @@ def mercurial(repo, dest, branch=None, r
     # the working directory directly from the repo, ignoring the sharing
     # settings
     if os.path.exists(dest):
         if not os.path.exists(os.path.join(dest, ".hg")):
             log.warning("%s doesn't appear to be a valid hg directory; clobbering", dest)
             remove_path(dest)
         elif not os.path.exists(os.path.join(dest, ".hg", "sharedpath")):
             try:
+                if autoPurge:
+                    purge(dest)
                 return pull(repo, dest, update_dest=update_dest, branch=branch,
                             revision=revision,
                             mirrors=mirrors)
             except subprocess.CalledProcessError:
                 log.warning("Error pulling changes into %s from %s; clobbering", dest, repo)
                 log.debug("Exception:", exc_info=True)
                 remove_path(dest)
 
@@ -420,18 +430,20 @@ def mercurial(repo, dest, branch=None, r
                 # Clobber!
                 log.info("We're currently shared from %s, but are being requested to pull from %s (%s); clobbering", dest_sharedPath_data, repo, norm_sharedRepo)
                 remove_path(dest)
 
         try:
             log.info("Updating shared repo")
             mercurial(repo, sharedRepo, branch=branch, revision=revision,
                       update_dest=False, shareBase=None, clone_by_rev=clone_by_rev,
-                      mirrors=mirrors, bundles=bundles)
+                      mirrors=mirrors, bundles=bundles, autoPurge=False)
             if os.path.exists(dest):
+                if autoPurge:
+                    purge(dest)
                 return update(dest, branch=branch, revision=revision)
 
             try:
                 log.info("Trying to share %s to %s", sharedRepo, dest)
                 return share(sharedRepo, dest, branch=branch, revision=revision)
             except subprocess.CalledProcessError:
                 if not allowUnsharedLocalClones:
                     # Re-raise the exception so it gets caught below.
new file mode 100644
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,25 @@
+[tox]
+envlist = py27
+
+[testenv]
+deps =
+    nose==1.3.0
+    coverage==3.6
+    pep8==1.4.3
+    jinja2==2.6
+    mock==1.0.1
+    webob==1.2.3
+    gevent==0.13.8
+    IPy==0.81
+
+
+setenv =
+    PYTHONPATH = {toxinidir}/lib/python/vendor/poster-0.8.1:{toxinidir}/lib/python:{toxinidir}/lib/python/vendor:$PYTHONPATH
+
+commands =
+    coverage erase
+    coverage run --branch --source {toxinidir}/lib/python {envbindir}/nosetests --with-xunit {toxinidir}/lib/python
+
+[pep8]
+max-line-length = 159
+exclude = vendor,.tox,