treemanifests: fix streaming clone
authorMartin von Zweigbergk <martinvonz@google.com>
Thu, 04 Feb 2016 08:34:07 -0800
changeset 29956 fb92927f977526795e9141e02141b589ebac902f
parent 29955 bb45190a54b754dafa66b24c2414d33401c0596e
child 29957 86c4cbdaffeeb75b9f0298774ce6b3db8d5aa0aa
push id175
push usergszorc@mozilla.com
push dateWed, 10 Feb 2016 01:24:28 +0000
treemanifests: fix streaming clone Similar to the previous patch, the .hg/store/meta/ directory does not get copied when when using "hg clone --uncompressed". Fix by including "meta/" in store.datafiles(). This seems safe to do, as there are only a few users of this method. "hg manifest" already filters the paths by "data/" prefix. The calls from largefiles also seem safe. The use in verify needs updating to prevent it from mistaking dirlogs for orphaned filelogs. That change is included in this patch. Since the dirlogs will now be in the fncache when using fncachestore, let's also update debugrebuildfncache(). That will also allow any existing treemanifest repos to get their dirlogs into the fncache. Also update test-treemanifest.t to use an a directory name that requires dot-encoding and uppercase-encoding so we test that the path encoding works.
mercurial/repair.py
mercurial/store.py
mercurial/verify.py
tests/run-tests.py
tests/test-treemanifest.t
--- a/mercurial/repair.py
+++ b/mercurial/repair.py
@@ -268,16 +268,26 @@ def rebuildfncache(ui, repo):
 
                 if repo.store._exists(i):
                     newentries.add(i)
                 if repo.store._exists(d):
                     newentries.add(d)
 
         ui.progress(_('changeset'), None)
 
+        if 'treemanifest' in repo: # safe but unnecessary otherwise
+            for dir in util.dirs(seenfiles):
+                i = 'meta/%s/00manifest.i' % dir
+                d = 'meta/%s/00manifest.d' % dir
+
+                if repo.store._exists(i):
+                    newentries.add(i)
+                if repo.store._exists(d):
+                    newentries.add(d)
+
         addcount = len(newentries - oldentries)
         removecount = len(oldentries - newentries)
         for p in sorted(oldentries - newentries):
             ui.write(_('removing %s\n') % p)
         for p in sorted(newentries - oldentries):
             ui.write(_('adding %s\n') % p)
 
         if addcount or removecount:
--- a/mercurial/store.py
+++ b/mercurial/store.py
@@ -325,17 +325,17 @@ class basicstore(object):
                         n = util.pconvert(fp[striplen:])
                         l.append((decodedir(n), n, st.st_size))
                     elif kind == stat.S_IFDIR and recurse:
                         visit.append(fp)
         l.sort()
         return l
 
     def datafiles(self):
-        return self._walk('data', True)
+        return self._walk('data', True) + self._walk('meta', True)
 
     def topfiles(self):
         # yield manifest before changelog
         return reversed(self._walk('', False))
 
     def walk(self):
         '''yields (unencoded, encoded, size)'''
         # yield data files first
@@ -373,17 +373,17 @@ class encodedstore(basicstore):
         self.path = vfs.base
         self.createmode = _calcmode(vfs)
         vfs.createmode = self.createmode
         self.rawvfs = vfs
         self.vfs = scmutil.filtervfs(vfs, encodefilename)
         self.opener = self.vfs
 
     def datafiles(self):
-        for a, b, size in self._walk('data', True):
+        for a, b, size in super(encodedstore, self).datafiles():
             try:
                 a = decodefilename(a)
             except KeyError:
                 a = None
             yield a, b, size
 
     def join(self, f):
         return self.path + '/' + encodefilename(f)
@@ -455,17 +455,18 @@ class fncache(object):
 
 class _fncachevfs(scmutil.abstractvfs, scmutil.auditvfs):
     def __init__(self, vfs, fnc, encode):
         scmutil.auditvfs.__init__(self, vfs)
         self.fncache = fnc
         self.encode = encode
 
     def __call__(self, path, mode='r', *args, **kw):
-        if mode not in ('r', 'rb') and path.startswith('data/'):
+        if mode not in ('r', 'rb') and (path.startswith('data/') or
+                                        path.startswith('meta/')):
             self.fncache.add(path)
         return self.vfs(self.encode(path), mode, *args, **kw)
 
     def join(self, path):
         if path:
             return self.vfs.join(self.encode(path))
         else:
             return self.vfs.join(path)
--- a/mercurial/verify.py
+++ b/mercurial/verify.py
@@ -279,17 +279,17 @@ class verifier(object):
         revlogv1 = self.revlogv1
         havemf = self.havemf
         ui.status(_("checking files\n"))
 
         storefiles = set()
         for f, f2, size in repo.store.datafiles():
             if not f:
                 self.err(None, _("cannot decode filename '%s'") % f2)
-            elif size > 0 or not revlogv1:
+            elif (size > 0 or not revlogv1) and f.startswith('data/'):
                 storefiles.add(_normpath(f))
 
         files = sorted(set(filenodes) | set(filelinkrevs))
         total = len(files)
         revisions = 0
         for i, f in enumerate(files):
             ui.progress(_('checking'), i, item=f, total=total)
             try:
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -714,16 +714,18 @@ class Test(unittest.TestCase):
         Test output needs to be normalized so it can be compared to expected
         output. This function defines how some of that normalization will
         occur.
         """
         r = [
             (br':%d\b' % self._startport, b':$HGPORT'),
             (br':%d\b' % (self._startport + 1), b':$HGPORT1'),
             (br':%d\b' % (self._startport + 2), b':$HGPORT2'),
+            (br':%d\b' % (self._startport + 2), b':$HGPORT3'),
+            (br':%d\b' % (self._startport + 2), b':$HGPORT4'),
             (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$',
              br'\1 (glob)'),
             ]
 
         if os.name == 'nt':
             r.append(
                 (b''.join(c.isalpha() and b'[%s%s]' % (c.lower(), c.upper()) or
                     c in b'/\\' and br'[/\\]' or c.isdigit() and c or b'\\' + c
@@ -736,16 +738,18 @@ class Test(unittest.TestCase):
     def _getenv(self):
         """Obtain environment variables to use during test execution."""
         env = os.environ.copy()
         env['TESTTMP'] = self._testtmp
         env['HOME'] = self._testtmp
         env["HGPORT"] = str(self._startport)
         env["HGPORT1"] = str(self._startport + 1)
         env["HGPORT2"] = str(self._startport + 2)
+        env["HGPORT3"] = str(self._startport + 3)
+        env["HGPORT4"] = str(self._startport + 4)
         env["HGRCPATH"] = os.path.join(self._threadtmp, b'.hgrc')
         env["DAEMON_PIDS"] = os.path.join(self._threadtmp, b'daemon.pids')
         env["HGEDITOR"] = ('"' + sys.executable + '"'
                            + ' -c "import sys; sys.exit(0)"')
         env["HGMERGE"] = "internal:merge"
         env["HGUSER"]   = "test"
         env["HGENCODING"] = "ascii"
         env["HGENCODINGMODE"] = "strict"
--- a/tests/test-treemanifest.t
+++ b/tests/test-treemanifest.t
@@ -362,44 +362,44 @@ Pushing to an empty repo works
   $ grep treemanifest clone/.hg/requires
   treemanifest
 
 Create deeper repo with tree manifests.
 
   $ hg --config experimental.treemanifest=True init deeprepo
   $ cd deeprepo
 
-  $ mkdir a
+  $ mkdir .A
   $ mkdir b
   $ mkdir b/bar
   $ mkdir b/bar/orange
   $ mkdir b/bar/orange/fly
   $ mkdir b/foo
   $ mkdir b/foo/apple
   $ mkdir b/foo/apple/bees
 
-  $ touch a/one.txt
-  $ touch a/two.txt
+  $ touch .A/one.txt
+  $ touch .A/two.txt
   $ touch b/bar/fruits.txt
   $ touch b/bar/orange/fly/gnat.py
   $ touch b/bar/orange/fly/housefly.txt
   $ touch b/foo/apple/bees/flower.py
   $ touch c.txt
   $ touch d.py
 
   $ hg ci -Aqm 'initial'
 
 We'll see that visitdir works by removing some treemanifest revlogs and running
 the files command with various parameters.
 
 Test files from the root.
 
   $ hg files -r .
-  a/one.txt (glob)
-  a/two.txt (glob)
+  .A/one.txt (glob)
+  .A/two.txt (glob)
   b/bar/fruits.txt (glob)
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
   b/foo/apple/bees/flower.py (glob)
   c.txt
   d.py
 
 Excludes with a glob should not exclude everything from the glob's root
@@ -407,57 +407,57 @@ Excludes with a glob should not exclude 
   $ hg files -r . -X 'b/fo?' b
   b/bar/fruits.txt (glob)
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
   $ cp -r .hg/store .hg/store-copy
 
 Test files for a subdirectory.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ hg files -r . b
   b/bar/fruits.txt (glob)
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
   b/foo/apple/bees/flower.py (glob)
   $ cp -rT .hg/store-copy .hg/store
 
 Test files with just includes and excludes.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/bar/orange/fly
   $ rm -r .hg/store/meta/b/foo/apple/bees
   $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X path:b/foo/apple/bees
   b/bar/fruits.txt (glob)
   $ cp -rT .hg/store-copy .hg/store
 
 Test files for a subdirectory, excluding a directory within it.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/foo
   $ hg files -r . -X path:b/foo b
   b/bar/fruits.txt (glob)
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
   $ cp -rT .hg/store-copy .hg/store
 
 Test files for a sub directory, including only a directory within it, and
 including an unrelated directory.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/foo
   $ hg files -r . -I path:b/bar/orange -I path:a b
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
   $ cp -rT .hg/store-copy .hg/store
 
 Test files for a pattern, including a directory, and excluding a directory
 within that.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/foo
   $ rm -r .hg/store/meta/b/bar/orange
   $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange
   b/bar/fruits.txt (glob)
   $ cp -rT .hg/store-copy .hg/store
 
 Add some more changes to the deep repo
   $ echo narf >> b/bar/fruits.txt
@@ -483,32 +483,32 @@ We can clone even with the knob turned o
 No server errors.
   $ cat deeprepo/errors.log
 requires got updated to include treemanifest
   $ cat deepclone/.hg/requires | grep treemanifest
   treemanifest
 Tree manifest revlogs exist.
   $ find deepclone/.hg/store/meta | sort
   deepclone/.hg/store/meta
-  deepclone/.hg/store/meta/a
-  deepclone/.hg/store/meta/a/00manifest.i
   deepclone/.hg/store/meta/b
   deepclone/.hg/store/meta/b/00manifest.i
   deepclone/.hg/store/meta/b/bar
   deepclone/.hg/store/meta/b/bar/00manifest.i
   deepclone/.hg/store/meta/b/bar/orange
   deepclone/.hg/store/meta/b/bar/orange/00manifest.i
   deepclone/.hg/store/meta/b/bar/orange/fly
   deepclone/.hg/store/meta/b/bar/orange/fly/00manifest.i
   deepclone/.hg/store/meta/b/foo
   deepclone/.hg/store/meta/b/foo/00manifest.i
   deepclone/.hg/store/meta/b/foo/apple
   deepclone/.hg/store/meta/b/foo/apple/00manifest.i
   deepclone/.hg/store/meta/b/foo/apple/bees
   deepclone/.hg/store/meta/b/foo/apple/bees/00manifest.i
+  deepclone/.hg/store/meta/~2e_a
+  deepclone/.hg/store/meta/~2e_a/00manifest.i
 Verify passes.
   $ cd deepclone
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
@@ -520,30 +520,38 @@ Create clones using old repo formats to 
   >   http://localhost:$HGPORT2 deeprepo-basicstore
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 3 changesets with 10 changes to 8 files
   updating to branch default
   8 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  $ grep store deeprepo-basicstore/.hg/requires
+  $ cd deeprepo-basicstore
+  $ grep store .hg/requires
   [1]
+  $ hg serve -p $HGPORT3 -d --pid-file=hg.pid --errorlog=errors.log
+  $ cat hg.pid >> $DAEMON_PIDS
+  $ cd ..
   $ hg clone --config format.usefncache=False \
   >   --config experimental.changegroup3=True \
   >   http://localhost:$HGPORT2 deeprepo-encodedstore
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 3 changesets with 10 changes to 8 files
   updating to branch default
   8 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  $ grep fncache deeprepo-encodedstore/.hg/requires
+  $ cd deeprepo-encodedstore
+  $ grep fncache .hg/requires
   [1]
+  $ hg serve -p $HGPORT4 -d --pid-file=hg.pid --errorlog=errors.log
+  $ cat hg.pid >> $DAEMON_PIDS
+  $ cd ..
 
 Local clone with basicstore
   $ hg clone -U deeprepo-basicstore local-clone-basicstore
   $ hg -R local-clone-basicstore verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
@@ -561,8 +569,60 @@ Local clone with encodedstore
 Local clone with fncachestore
   $ hg clone -U deeprepo local-clone-fncachestore
   $ hg -R local-clone-fncachestore verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
+
+Stream clone with basicstore
+  $ hg clone --config experimental.changegroup3=True --uncompressed -U \
+  >   http://localhost:$HGPORT3 stream-clone-basicstore
+  streaming all changes
+  18 files to transfer, * of data (glob)
+  transferred * in * seconds (*) (glob)
+  searching for changes
+  no changes found
+  $ hg -R stream-clone-basicstore verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+
+Stream clone with encodedstore
+  $ hg clone --config experimental.changegroup3=True --uncompressed -U \
+  >   http://localhost:$HGPORT4 stream-clone-encodedstore
+  streaming all changes
+  18 files to transfer, * of data (glob)
+  transferred * in * seconds (*) (glob)
+  searching for changes
+  no changes found
+  $ hg -R stream-clone-encodedstore verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+
+Stream clone with fncachestore
+  $ hg clone --config experimental.changegroup3=True --uncompressed -U \
+  >   http://localhost:$HGPORT2 stream-clone-fncachestore
+  streaming all changes
+  18 files to transfer, * of data (glob)
+  transferred * in * seconds (*) (glob)
+  searching for changes
+  no changes found
+  $ hg -R stream-clone-fncachestore verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+
+Packed bundle
+  $ hg -R deeprepo debugcreatestreamclonebundle repo-packed.hg
+  writing 3349 bytes for 18 files
+  bundle requirements: generaldelta, revlogv1, treemanifest
+  $ hg debugbundle --spec repo-packed.hg
+  none-packed1;requirements%3Dgeneraldelta%2Crevlogv1%2Ctreemanifest