Bug 1466514 - download the uploaded wpt-manifest, r=jgraham
authorAhilya Sinha <asinha@mozilla.com>
Mon, 04 Jun 2018 18:16:28 +0530
changeset 421583 e79ad2cef7ad
parent 421582 ebecd80e1816
child 421584 997adbec4c49
push id64922
push userjames@hoppipolla.co.uk
push dateWed, 06 Jun 2018 14:22:39 +0000
treeherderautoland@e79ad2cef7ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1466514
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1466514 - download the uploaded wpt-manifest, r=jgraham MozReview-Commit-ID: BP4A9S5xG7M
testing/web-platform/mach_commands.py
testing/web-platform/manifestdownload.py
testing/web-platform/vcs.py
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -286,16 +286,25 @@ class WPTManifestUpdater(MozbuildObject)
     def run_update(self, check_clean=False, rebuild=False, **kwargs):
         import manifestupdate
         from wptrunner import wptlogging
         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
         wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
         manifestupdate.update(logger, wpt_dir, check_clean, rebuild)
 
 
+class WPTManifestDownloader(MozbuildObject):
+    def run_download(self, path=None, tests_root=None, force=False, **kwargs):
+        import manifestdownload
+        from wptrunner import wptlogging
+        logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
+        wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
+        manifestdownload.run(logger, wpt_dir, self.topsrcdir, force)
+
+
 def create_parser_update():
     from update import updatecommandline
     return updatecommandline.create_parser()
 
 def create_parser_reduce():
     from wptrunner import wptcommandline
     return wptcommandline.create_parser_reduce()
 
@@ -319,16 +328,20 @@ def create_parser_create():
     p.add_argument("path", action="store", help="Path to the test file")
     return p
 
 
 def create_parser_manifest_update():
     import manifestupdate
     return manifestupdate.create_parser()
 
+def create_parser_manifest_download():
+    import manifestdownload
+    return manifestdownload.create_parser()
+
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     def setup(self):
         self._activate_virtualenv()
 
     @Command("web-platform-tests",
              category="testing",
@@ -402,8 +415,17 @@ class MachCommands(MachCommandBase):
 
     @Command("wpt-manifest-update",
              category="testing",
              parser=create_parser_manifest_update)
     def wpt_manifest_update(self, **params):
         self.setup()
         wpt_manifest_updater = self._spawn(WPTManifestUpdater)
         return wpt_manifest_updater.run_update(**params)
+
+
+    @Command("wpt-manifest-download",
+             category="testing",
+             parser=create_parser_manifest_download)
+    def wpt_manifest_download(self, **params):
+        self.setup()
+        wpt_manifest_downloader = self._spawn(WPTManifestDownloader)
+        return wpt_manifest_downloader.run_download(**params)
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/manifestdownload.py
@@ -0,0 +1,130 @@
+from __future__ import absolute_import
+
+import argparse
+import json
+import io
+import os
+from datetime import datetime, timedelta
+import gzip
+from vcs import Mercurial
+import requests
+
+from six.moves.urllib.request import urlopen
+
+def abs_path(path):
+    return os.path.abspath(os.path.expanduser(path))
+
+
+def hg_commits(repo_root):
+    hg = Mercurial.get_func(repo_root)
+    return [item for item in hg("log", "-fl50", "--template={node}\n",
+            "testing/web-platform/tests/", "testing/web-platform/mozilla/tests").split("\n")
+            if item]
+
+
+def should_download(logger, manifest_path, rebuild_time=timedelta(days=5)):
+    # TODO: Improve logic for when to download. Maybe if x revisions behind?
+    if not os.path.exists(manifest_path):
+        return True
+    mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
+    if mtime < datetime.now() - rebuild_time:
+        return True
+    logger.info("Skipping manifest download because existing file is recent")
+    return False
+
+
+def taskcluster_url(logger, commits):
+    cset_url = ('https://hg.mozilla.org/mozilla-central/json-pushes?'
+                'changeset={changeset}&version=2&tipsonly=1')
+
+    tc_url = ('https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central.'
+              'revision.{changeset}.source.manifest-upload')
+
+    for revision in commits:
+        req = requests.get(cset_url.format(changeset=revision),
+                           headers={'Accept': 'application/json'})
+
+        req.raise_for_status()
+
+        result = req.json()
+        [cset] = result['pushes'].values()[0]['changesets']
+        resp = requests.get(tc_url.format(changeset=cset))
+
+        if req.status_code == 200:
+            return tc_url.format(changeset=cset)
+
+    logger.info("Can't find a commit-specific manifest so just using the most"
+                "recent one")
+
+    return ("https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central."
+            "latest.source.manifest-upload")
+
+
+def download_manifest(logger, wpt_dir, commits_func, url_func, force=False):
+    if not force and not should_download(logger, wpt_dir):
+        return False
+
+    commits = commits_func()
+    url = url_func(logger, commits) + "/artifacts/public/"
+
+    man_url= url + "manifest.json.gz"
+    moz_man_url= url + "moz_manifest.json.gz"
+
+    return ( _download(logger, os.path.join(wpt_dir, "meta", "MANIFEST.json"), man_url) and
+             _download(logger, os.path.join(wpt_dir,"mozilla", "meta", "MANIFEST.json"), moz_man_url))
+
+
+def _download(logger, manifest_path, url):
+    if not url:
+        logger.warning("No generated manifest found")
+        return False
+
+    logger.info("Downloading manifest from %s" % url)
+    try:
+        resp = urlopen(url)
+    except Exception:
+        logger.warning("Downloading pregenerated manifest failed")
+        return False
+
+    if resp.code != 200:
+        logger.warning("Downloading pregenerated manifest failed; got"
+                        "HTTP status %d" % resp.code)
+        return False
+
+    gzf = gzip.GzipFile(fileobj=io.BytesIO(resp.read()))
+
+    try:
+        decompressed = gzf.read()
+    except IOError:
+        logger.warning("Failed to decompress downloaded file")
+        return False
+
+    try:
+        with open(manifest_path, 'wb') as f:
+            f.write(decompressed)
+    except Exception as e:
+        logger.warning("Failed to write manifest at %s" % manifest_path)
+        return False
+
+    logger.info("Manifest at %s downloaded" % manifest_path)
+    return True
+
+
+def create_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-p", "--path", type=abs_path, help="Path to manifest file.")
+    parser.add_argument(
+        "--force", action="store_true",
+        help="Always download, even if the existing manifest is recent")
+    return parser
+
+
+def download_from_taskcluster(logger, wpt_dir, repo_root, force=False):
+    return download_manifest(logger, wpt_dir, lambda: hg_commits(repo_root),
+                             taskcluster_url, force)
+
+
+def run(logger, wpt_dir, repo_root, force=False):
+    success = download_from_taskcluster(logger, wpt_dir, repo_root, force)
+    return 0 if success else 1
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/vcs.py
@@ -0,0 +1,20 @@
+import os
+import subprocess
+
+class Mercurial(object):
+
+    def __init__(self, repo_root):
+        self.root = os.path.abspath(repo_root)
+        self.hg = Mercurial.get_func(repo_root)
+
+
+    @staticmethod
+    def get_func(repo_path):
+        def hg(cmd, *args):
+            full_cmd = ["hg", cmd] + list(args)
+            try:
+                return subprocess.check_output(full_cmd, cwd=repo_path, stderr=subprocess.STDOUT)
+            except Exception as e:
+                raise(e)
+            # TODO: Test on Windows.
+        return hg