streamclone: add support for bundle2 based stream clone
authorBoris Feld <boris.feld@octobus.net>
Wed, 17 Jan 2018 16:41:44 +0100
changeset 41649 7eedbd5d4880f6c0518e5ff706450a7389acb70d
parent 41648 40df727b6f4f3caceac92145905d37591eeaa46f
child 41650 b6ffd419463943dbc614c89fd9735dba37310254
push id653
push usergszorc@mozilla.com
push dateSun, 21 Jan 2018 00:53:23 +0000
streamclone: add support for bundle2 based stream clone The feature put to use the various bits introduced previously. If the server supports it, the client will request its stream clone through bundle2 instead of the legacy 'stream_out' commands. The bundle2 version use the better 'v2' version of stream bundles. The 'v2' format is not finalized yet. Now that there are some code running it, we can start working on it again. Performance numbers are available at the end of this series.
mercurial/bundle2.py
mercurial/configitems.py
mercurial/exchange.py
tests/test-clone-uncompressed.t
--- a/mercurial/bundle2.py
+++ b/mercurial/bundle2.py
@@ -1482,16 +1482,17 @@ capabilities = {'HG20': (),
                 'error': ('abort', 'unsupportedcontent', 'pushraced',
                           'pushkey'),
                 'listkeys': (),
                 'pushkey': (),
                 'digests': tuple(sorted(util.DIGESTS.keys())),
                 'remote-changegroup': ('http', 'https'),
                 'hgtagsfnodes': (),
                 'phases': ('heads',),
+                'stream': ('v2',),
                }
 
 def getrepocaps(repo, allowpushback=False):
     """return the bundle2 capabilities for a given repo
 
     Exists to allow extensions (like evolution) to mutate the capabilities.
     """
     caps = capabilities.copy()
@@ -1502,16 +1503,18 @@ def getrepocaps(repo, allowpushback=Fals
         caps['obsmarkers'] = supportedformat
     if allowpushback:
         caps['pushback'] = ()
     cpmode = repo.ui.config('server', 'concurrent-push-mode')
     if cpmode == 'check-related':
         caps['checkheads'] = ('related',)
     if 'phases' in repo.ui.configlist('devel', 'legacy.exchange'):
         caps.pop('phases')
+    if not repo.ui.configbool('experimental', 'bundle2.stream'):
+        caps.pop('stream')
     return caps
 
 def bundle2caps(remote):
     """return the bundle capabilities of a peer as dict"""
     raw = remote.capable('bundle2')
     if not raw and raw != '':
         return {}
     capsblob = urlreq.unquote(remote.capable('bundle2'))
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -426,16 +426,19 @@ coreconfigitem('experimental', 'bundle2-
     default=True,
 )
 coreconfigitem('experimental', 'bundle2-output-capture',
     default=False,
 )
 coreconfigitem('experimental', 'bundle2.pushback',
     default=False,
 )
+coreconfigitem('experimental', 'bundle2.stream',
+    default=False,
+)
 coreconfigitem('experimental', 'bundle2lazylocking',
     default=False,
 )
 coreconfigitem('experimental', 'bundlecomplevel',
     default=None,
 )
 coreconfigitem('experimental', 'changegroup3',
     default=False,
--- a/mercurial/exchange.py
+++ b/mercurial/exchange.py
@@ -1450,23 +1450,28 @@ def _pullbundle2(pullop):
     For now, the only supported data are changegroup."""
     kwargs = {'bundlecaps': caps20to10(pullop.repo)}
 
     # make ui easier to access
     ui = pullop.repo.ui
 
     # At the moment we don't do stream clones over bundle2. If that is
     # implemented then here's where the check for that will go.
-    streaming = False
+    streaming = streamclone.canperformstreamclone(pullop, bundle2=True)[0]
 
     # declare pull perimeters
     kwargs['common'] = pullop.common
     kwargs['heads'] = pullop.heads or pullop.rheads
 
-    if True:
+    if streaming:
+        kwargs['cg'] = False
+        kwargs['stream'] = True
+        pullop.stepsdone.add('changegroup')
+
+    else:
         # pulling changegroup
         pullop.stepsdone.add('changegroup')
 
         kwargs['cg'] = pullop.fetch
 
     legacyphase = 'phases' in ui.configlist('devel', 'legacy.exchange')
     hasbinaryphase = 'heads' in pullop.remotebundle2caps.get('phases', ())
     if (not legacyphase and hasbinaryphase):
--- a/tests/test-clone-uncompressed.t
+++ b/tests/test-clone-uncompressed.t
@@ -1,10 +1,19 @@
 #require serve
 
+#testcases stream-legacy stream-bundle2
+
+#if stream-bundle2
+  $ cat << EOF >> $HGRCPATH
+  > [experimental]
+  > bundle2.stream = yes
+  > EOF
+#endif
+
 Initialize repository
 the status call is to check for issue5130
 
   $ hg init server
   $ cd server
   $ touch foo
   $ hg -q commit -A -m initial
   >>> for i in range(1024):
@@ -13,34 +22,51 @@ the status call is to check for issue513
   $ hg -q commit -A -m 'add a lot of files'
   $ hg st
   $ hg serve -p $HGPORT -d --pid-file=hg.pid
   $ cat hg.pid >> $DAEMON_PIDS
   $ cd ..
 
 Basic clone
 
+#if stream-legacy
   $ hg clone --stream -U http://localhost:$HGPORT clone1
   streaming all changes
   1027 files to transfer, 96.3 KB of data
   transferred 96.3 KB in * seconds (*/sec) (glob)
   searching for changes
   no changes found
+#endif
+#if stream-bundle2
+  $ hg clone --stream -U http://localhost:$HGPORT clone1
+  streaming all changes
+  1027 files to transfer, 96.3 KB of data
+  transferred 96.3 KB in * seconds (* */sec) (glob)
+#endif
 
 --uncompressed is an alias to --stream
 
+#if stream-legacy
   $ hg clone --uncompressed -U http://localhost:$HGPORT clone1-uncompressed
   streaming all changes
   1027 files to transfer, 96.3 KB of data
   transferred 96.3 KB in * seconds (*/sec) (glob)
   searching for changes
   no changes found
+#endif
+#if stream-bundle2
+  $ hg clone --uncompressed -U http://localhost:$HGPORT clone1-uncompressed
+  streaming all changes
+  1027 files to transfer, 96.3 KB of data
+  transferred 96.3 KB in * seconds (* */sec) (glob)
+#endif
 
 Clone with background file closing enabled
 
+#if stream-legacy
   $ hg --debug --config worker.backgroundclose=true --config worker.backgroundcloseminfilecount=1 clone --stream -U http://localhost:$HGPORT clone-background | grep -v adding
   using http://localhost:$HGPORT/
   sending capabilities command
   sending branchmap command
   streaming all changes
   sending stream_out command
   1027 files to transfer, 96.3 KB of data
   starting 4 threads for background file closing
@@ -52,16 +78,38 @@ Clone with background file closing enabl
   no changes found
   sending getbundle command
   bundle2-input-bundle: with-transaction
   bundle2-input-part: "listkeys" (params: 1 mandatory) supported
   bundle2-input-part: "phase-heads" supported
   bundle2-input-part: total payload size 24
   bundle2-input-bundle: 1 parts total
   checking for updated bookmarks
+#endif
+#if stream-bundle2
+  $ hg --debug --config worker.backgroundclose=true --config worker.backgroundcloseminfilecount=1 clone --stream -U http://localhost:$HGPORT clone-background | grep -v adding
+  using http://localhost:$HGPORT/
+  sending capabilities command
+  query 1; heads
+  sending batch command
+  streaming all changes
+  sending getbundle command
+  bundle2-input-bundle: with-transaction
+  bundle2-input-part: "stream" (params: 4 mandatory) supported
+  applying stream bundle
+  1027 files to transfer, 96.3 KB of data
+  starting 4 threads for background file closing
+  transferred 96.3 KB in * seconds (* */sec) (glob)
+  bundle2-input-part: total payload size 110887
+  bundle2-input-part: "listkeys" (params: 1 mandatory) supported
+  bundle2-input-part: "phase-heads" supported
+  bundle2-input-part: total payload size 24
+  bundle2-input-bundle: 2 parts total
+  checking for updated bookmarks
+#endif
 
 Cannot stream clone when there are secret changesets
 
   $ hg -R server phase --force --secret -r tip
   $ hg clone --stream -U http://localhost:$HGPORT secret-denied
   warning: stream clone requested but server has them disabled
   requesting all changes
   adding changesets
@@ -74,22 +122,30 @@ Cannot stream clone when there are secre
 
 Streaming of secrets can be overridden by server config
 
   $ cd server
   $ hg serve --config server.uncompressedallowsecret=true -p $HGPORT -d --pid-file=hg.pid
   $ cat hg.pid > $DAEMON_PIDS
   $ cd ..
 
+#if stream-legacy
   $ hg clone --stream -U http://localhost:$HGPORT secret-allowed
   streaming all changes
   1027 files to transfer, 96.3 KB of data
   transferred 96.3 KB in * seconds (*/sec) (glob)
   searching for changes
   no changes found
+#endif
+#if stream-bundle2
+  $ hg clone --stream -U http://localhost:$HGPORT secret-allowed
+  streaming all changes
+  1027 files to transfer, 96.3 KB of data
+  transferred 96.3 KB in * seconds (* */sec) (glob)
+#endif
 
   $ killdaemons.py
 
 Verify interaction between preferuncompressed and secret presence
 
   $ cd server
   $ hg serve --config server.preferuncompressed=true -p $HGPORT -d --pid-file=hg.pid
   $ cat hg.pid > $DAEMON_PIDS
@@ -181,18 +237,28 @@ Stream repository with bookmarks
   $ hg -R server phase --draft 'secret()'
 
 add a bookmark
 
   $ hg -R server bookmark -r tip some-bookmark
 
 clone it
 
+#if stream-legacy
   $ hg clone --stream http://localhost:$HGPORT with-bookmarks
   streaming all changes
   1027 files to transfer, 96.3 KB of data
   transferred 96.3 KB in * seconds (*) (glob)
   searching for changes
   no changes found
   updating to branch default
   1025 files updated, 0 files merged, 0 files removed, 0 files unresolved
+#endif
+#if stream-bundle2
+  $ hg clone --stream http://localhost:$HGPORT with-bookmarks
+  streaming all changes
+  1027 files to transfer, 96.3 KB of data
+  transferred 96.3 KB in * seconds (* */sec) (glob)
+  updating to branch default
+  1025 files updated, 0 files merged, 0 files removed, 0 files unresolved
+#endif
   $ hg -R with-bookmarks bookmarks
      some-bookmark             1:c17445101a72