hgmo: replaces filesdata wire proto command with working version (bug 1513077); r=sheehan
authorGregory Szorc <gps@mozilla.com>
Tue, 11 Dec 2018 01:11:01 +0000
changeset 6785 ec53cd7868c1
parent 6784 fc6ee016a017
child 6786 8af6070012c0
push id3372
push usergszorc@mozilla.com
push dateTue, 11 Dec 2018 16:50:25 +0000
treeherderversion-control-tools@ec53cd7868c1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssheehan
bugs1513077
hgmo: replaces filesdata wire proto command with working version (bug 1513077); r=sheehan We copy the command handler and supporting function from revision 08cfa77d7288 of the Mercurial repository in order to work around bugs related to revision selection and linknodes. The test changes demonstrate that this has the intended effect. Differential Revision: https://phabricator.services.mozilla.com/D14111
hgext/hgmo/__init__.py
hgext/hgmo/tests/test-wireproto-command-filesdata.t
--- a/hgext/hgmo/__init__.py
+++ b/hgext/hgmo/__init__.py
@@ -69,16 +69,17 @@ message being returned to the user. stde
 Wrapper commands typically read arguments, set up a secure execution
 environment, then invoke the relevant mozbuild APIs to obtain requested info.
 
 Wrapper commands may invoke ``hg mozbuildinfo --pipemode`` to retrieve
 moz.build info. In fact, the wrapper command itself can be defined as this
 string. Of course, no security will be provided.
 """
 
+import collections
 import hashlib
 import json
 import os
 import subprocess
 import types
 
 from mercurial.i18n import _
 from mercurial.node import bin, short
@@ -1011,16 +1012,116 @@ def mozrawcachefiles(repo, proto, files)
             b'location': location,
             b'path': name,
             b'size': len(data),
         }
 
         yield wireprototypes.indefinitebytestringresponse([data])
 
 
+# TRACKING hg49 filesdata command in 4.8 has bugs. We work around with custom
+# command implementation.
+
+def emitfilerevisions(repo, path, revisions, linknodes, fields):
+    for revision in revisions:
+        d = {
+            b'node': revision.node,
+        }
+
+        if b'parents' in fields:
+            d[b'parents'] = [revision.p1node, revision.p2node]
+
+        if b'linknode' in fields:
+            d[b'linknode'] = linknodes[revision.node]
+
+        followingmeta = []
+        followingdata = []
+
+        if b'revision' in fields:
+            if revision.revision is not None:
+                followingmeta.append((b'revision', len(revision.revision)))
+                followingdata.append(revision.revision)
+            else:
+                d[b'deltabasenode'] = revision.basenode
+                followingmeta.append((b'delta', len(revision.delta)))
+                followingdata.append(revision.delta)
+
+        if followingmeta:
+            d[b'fieldsfollowing'] = followingmeta
+
+        yield d
+
+        for extra in followingdata:
+            yield extra
+
+
+def filesdata(repo, proto, haveparents, fields, pathfilter, revisions):
+    # TODO This should operate on a repo that exposes obsolete changesets. There
+    # is a race between a client making a push that obsoletes a changeset and
+    # another client fetching files data for that changeset. If a client has a
+    # changeset, it should probably be allowed to access files data for that
+    # changeset.
+
+    outgoing = wireprotov2server.resolvenodes(repo, revisions)
+    filematcher = wireprotov2server.makefilematcher(repo, pathfilter)
+
+    # path -> {fnode: linknode}
+    fnodes = collections.defaultdict(dict)
+
+    # We collect the set of relevant file revisions by iterating the changeset
+    # revisions and either walking the set of files recorded in the changeset
+    # or by walking the manifest at that revision. There is probably room for a
+    # storage-level API to request this data, as it can be expensive to compute
+    # and would benefit from caching or alternate storage from what revlogs
+    # provide.
+    for node in outgoing:
+        ctx = repo[node]
+        mctx = ctx.manifestctx()
+        md = mctx.read()
+
+        if haveparents:
+            checkpaths = ctx.files()
+        else:
+            checkpaths = md.keys()
+
+        for path in checkpaths:
+            fnode = md[path]
+
+            if path in fnodes and fnode in fnodes[path]:
+                continue
+
+            if not filematcher(path):
+                continue
+
+            fnodes[path].setdefault(fnode, node)
+
+    yield {
+        b'totalpaths': len(fnodes),
+        b'totalitems': sum(len(v) for v in fnodes.values())
+    }
+
+    for path, filenodes in sorted(fnodes.items()):
+        try:
+            store = wireprotov2server.getfilestore(repo, proto, path)
+        except wireprotov2server.FileAccessError as e:
+            raise error.WireprotoCommandError(e.msg, e.args)
+
+        yield {
+            b'path': path,
+            b'totalitems': len(filenodes),
+        }
+
+        revisions = store.emitrevisions(filenodes.keys(),
+                                        revisiondata=b'revision' in fields,
+                                        assumehaveparentrevisions=haveparents)
+
+        for o in emitfilerevisions(repo, path, revisions, filenodes, fields):
+            yield o
+
+
 def rawstorefiledata_cache_fn(repo, proto, cacher, **args):
     # Only cache if we request changelog + manifestlog with no path filter.
     # Caching is hard and restricting what is cached is safer.
     if set(args.get('files', [])) != {'changelog', 'manifestlog'}:
         return None
 
     if args.get('pathfilter'):
         return None
@@ -1101,16 +1202,19 @@ def extsetup(ui):
     webcommands.__all__.append('automationrelevance')
 
     setattr(webcommands, 'isancestor', isancestorwebcommand)
     webcommands.__all__.append('isancestor')
 
     setattr(webcommands, 'repoinfo', repoinfowebcommand)
     webcommands.__all__.append('repoinfo')
 
+    # TRACKING hg49 install custom filesdata command handler to work around bugs.
+    wireprotov2server.COMMANDS[b'filesdata'].func = filesdata
+
     # Teach rawstorefiledata command to cache.
     wireprotov2server.COMMANDS[b'rawstorefiledata'].cachekeyfn = rawstorefiledata_cache_fn
 
 
 def reposetup(ui, repo):
     fasupport = import_module('hgext.fastannotate.support')
 
     if not fasupport:
--- a/hgext/hgmo/tests/test-wireproto-command-filesdata.t
+++ b/hgext/hgmo/tests/test-wireproto-command-filesdata.t
@@ -112,17 +112,17 @@ rewritten accordingly
       b'totalitems': 1,
       b'totalpaths': 1
     },
     {
       b'path': b'dupe-file',
       b'totalitems': 1
     },
     {
-      b'linknode': b'\x16\x81\xc3?\x9f\x80Q\xcf\xfd\x8c#\xa6\xd4\x08\x18-\xf8\xf3\xa1f',
+      b'linknode': b'c\x9c\x89\x90\xd6\xa5l\xbc\x951\xfbY@\xdb\x0e$8\xf13b',
       b'node': b'.\xd2\xa3\x91*\x0b$P C\xea\xe8N\xe4\xb2y\xc1\x8b\x90\xdd'
     }
   ]
 
   $ hg debugwireproto --nologhandshake --peer http2 http://$LOCALIP:$HGPORT/ << EOF
   > command filesdata
   >     revisions eval:[{
   >         b'type': b'changesetexplicit',
@@ -132,14 +132,22 @@ rewritten accordingly
   >     fields eval:[b'linknode']
   >     haveparents eval:True
   >     pathfilter eval:{b'include': [b'path:dupe-file']}
   > EOF
   creating http peer for wire protocol version 2
   sending filesdata command
   response: gen[
     {
-      b'totalitems': 0,
-      b'totalpaths': 0
+      b'totalitems': 1,
+      b'totalpaths': 1
+    },
+    {
+      b'path': b'dupe-file',
+      b'totalitems': 1
+    },
+    {
+      b'linknode': b'c\x9c\x89\x90\xd6\xa5l\xbc\x951\xfbY@\xdb\x0e$8\xf13b',
+      b'node': b'.\xd2\xa3\x91*\x0b$P C\xea\xe8N\xe4\xb2y\xc1\x8b\x90\xdd'
     }
   ]
 
   $ cat error.log