bug 1540655: build, remote: add mach command for vendoring Puppeteer; r=firefox-build-system-reviewers,chmanchester draft
authorpulselistener
Fri, 16 Aug 2019 13:21:57 +0000
changeset 2220726 58122219d1c1efbb80bd1c4b637edef0372684f8
parent 2220725 57f4070b436680417f9a634d9258fab593b271c9
child 2220727 2936da435c51787d88e2eb0ff6c3787c431593d1
push id406994
push userreviewbot
push dateFri, 16 Aug 2019 13:23:14 +0000
treeherdertry@5f50de08beda [default view] [failures only]
reviewersfirefox-build-system-reviewers, chmanchester
bugs1540655
milestone70.0a1
bug 1540655: build, remote: add mach command for vendoring Puppeteer; r=firefox-build-system-reviewers,chmanchester Introduces "./mach remote vendor-puppeteer" for vendoring the Puppeteer client without dependencies into remote/test/puppeteer/. The particular checkout of Puppeteer is https://github.com/andreastt/puppeteer/tree/firefox, which contains a couple of hotfixes we need for the client to work with the Firefox implementation of CDP. The remote agent targets a specific version of Puppeteer, so it is not suitable for this to be vendored under third_party/. We also wouldn't want other code in central to accidentally use a patched fork. The vendoring process is not part of "./mach vendor" because it does not yet have Node.js support, and implementing that for mach is outside the scope of getting the Puppeteer tests running with the remote agent. Differential Revision: https://phabricator.services.mozilla.com/D37007 Differential Diff: PHID-DIFF-4kbopizbrjcih62lm27c
.eslintignore
build/mach_bootstrap.py
remote/mach_commands.py
--- a/.eslintignore
+++ b/.eslintignore
@@ -254,16 +254,17 @@ python/**
 # Remote agent
 remote/Protocol.jsm
 remote/server/HTTPD.jsm
 remote/server/Packet.jsm
 remote/server/Socket.jsm
 remote/server/Stream.jsm
 remote/test/browser/chrome-remote-interface.js
 remote/test/demo.js
+remote/test/puppeteer/
 
 # NSS / taskcluster only.
 security/nss/**
 
 # services/ exclusions
 
 # Third party services
 services/common/kinto-http-client.js
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -49,16 +49,17 @@ MACH_MODULES = [
     'python/mozbuild/mozbuild/backend/mach_commands.py',
     'python/mozbuild/mozbuild/build_commands.py',
     'python/mozbuild/mozbuild/code-analysis/mach_commands.py',
     'python/mozbuild/mozbuild/compilation/codecomplete.py',
     'python/mozbuild/mozbuild/frontend/mach_commands.py',
     'python/mozbuild/mozbuild/mach_commands.py',
     'python/mozrelease/mozrelease/mach_commands.py',
     'python/safety/mach_commands.py',
+    'remote/mach_commands.py',
     'taskcluster/mach_commands.py',
     'testing/awsy/mach_commands.py',
     'testing/firefox-ui/mach_commands.py',
     'testing/geckodriver/mach_commands.py',
     'testing/mach_commands.py',
     'testing/marionette/mach_commands.py',
     'testing/mochitest/mach_commands.py',
     'testing/mozharness/mach_commands.py',
new file mode 100644
--- /dev/null
+++ b/remote/mach_commands.py
@@ -0,0 +1,160 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+)
+
+import sys
+import os
+import tempfile
+import shutil
+import subprocess
+
+from mach.decorators import (
+    Command,
+    CommandArgument,
+    CommandProvider,
+    SubCommand,
+)
+
+from mozbuild.base import MachCommandBase
+
+
+EX_CONFIG = 78
+EX_SOFTWARE = 70
+EX_USAGE = 64
+
+DEFAULT_REPO = "https://github.com/andreastt/puppeteer.git"
+DEFAULT_COMMITISH = "firefox"
+
+
+@CommandProvider
+class RemoteCommands(MachCommandBase):
+    def __init__(self, context):
+        MachCommandBase.__init__(self, context)
+        self.remotedir = os.path.join(self.topsrcdir, "remote")
+
+    @Command("remote", category="misc",
+             description="Remote protocol related operations.")
+    def remote(self):
+        self.parser.print_usage()
+        exit(EX_USAGE)
+
+    @SubCommand("remote", "vendor-puppeteer",
+                "Pull in latest changes of the Puppeteer client.")
+    @CommandArgument("--repository",
+                     metavar="REPO",
+                     default=DEFAULT_REPO,
+                     help="The (possibly remote) repository to clone from. "
+                          "Defaults to {}.".format(DEFAULT_REPO))
+    @CommandArgument("--commitish",
+                     metavar="COMMITISH",
+                     default=DEFAULT_COMMITISH,
+                     help="The commit or tag object name to check out. "
+                          "Defaults to \"{}\".".format(DEFAULT_COMMITISH))
+    def vendor_puppeteer(self, repository, commitish):
+        puppeteerdir = os.path.join(self.remotedir, "test", "puppeteer")
+
+        shutil.rmtree(puppeteerdir, ignore_errors=True)
+        os.makedirs(puppeteerdir)
+        with TemporaryDirectory() as tmpdir:
+            git("clone", "-q", repository, tmpdir)
+            git("checkout", commitish, worktree=tmpdir)
+            git("checkout-index", "-a", "-f",
+                "--prefix", "{}/".format(puppeteerdir),
+                worktree=tmpdir)
+
+        # remove files which may interfere with git checkout of central
+        try:
+            os.remove(os.path.join(puppeteerdir, ".gitattributes"))
+            os.remove(os.path.join(puppeteerdir, ".gitignore"))
+        except OSError:
+            pass
+
+        import yaml
+        annotation = {
+            "schema": 1,
+            "bugzilla": {
+                "product": "Remote Protocol",
+                "component": "Agent",
+            },
+            "origin": {
+                "name": "puppeteer",
+                "description": "Headless Chrome Node API",
+                "url": repository,
+                "license": "Apache-2.0",
+                "release": commitish,
+            },
+        }
+        with open(os.path.join(puppeteerdir, "moz.yaml"), "w") as fh:
+            yaml.safe_dump(annotation, fh,
+                           default_flow_style=False,
+                           encoding="utf-8",
+                           allow_unicode=True)
+
+
+def git(*args, **kwargs):
+    cmd = ("git",)
+    if kwargs.get("worktree"):
+        cmd += ("-C", kwargs["worktree"])
+    cmd += args
+
+    pipe = kwargs.get("pipe")
+    git_p = subprocess.Popen(cmd,
+                             env={"GIT_CONFIG_NOSYSTEM": "1"},
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+    pipe_p = None
+    if pipe:
+        pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
+
+    if pipe:
+        _, pipe_err = pipe_p.communicate()
+    out, git_err = git_p.communicate()
+
+    # use error from first program that failed
+    if git_p.returncode > 0:
+        exit(EX_SOFTWARE, git_err)
+    if pipe and pipe_p.returncode > 0:
+        exit(EX_SOFTWARE, pipe_err)
+
+    return out
+
+
+# tempfile.TemporaryDirectory missing from Python 2.7
+class TemporaryDirectory(object):
+    def __init__(self):
+        self.path = tempfile.mkdtemp()
+        self._closed = False
+
+    def __repr__(self):
+        return "<{} {!r}>".format(self.__class__.__name__, self.path)
+
+    def __enter__(self):
+        return self.path
+
+    def __exit__(self, exc, value, tb):
+        self.clean()
+
+    def __del__(self):
+        self.clean()
+
+    def clean(self):
+        if self.path and not self._closed:
+            shutil.rmtree(self.path)
+            self._closed = True
+
+
+def exit(code, error=None):
+    if error is not None:
+        if isinstance(error, Exception):
+            import traceback
+            traceback.print_exc()
+        else:
+            message = str(error).split("\n")[0].strip()
+            print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
+    sys.exit(code)