Bug 1509526 [wpt PR 14208] - Revert "Find manifest for download by tags instead of commits", a=testonly
authorPhilip Jägenstedt <philip@foolip.org>
Fri, 30 Nov 2018 16:54:12 +0000
changeset 449990 18b7ae06e7800497ead84ac4a271867b740d52f5
parent 449989 c63dfb6718227396054db673c0e8393c7c3ae36f
child 449991 5c242afe0b75850a4f099399c9fd172cc2b105a6
push id35189
push userccoroiu@mozilla.com
push dateTue, 11 Dec 2018 21:33:05 +0000
treeherdermozilla-central@ac7f3beb6333 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1509526, 14208, 14207
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1509526 [wpt PR 14208] - Revert "Find manifest for download by tags instead of commits", a=testonly Automatic update from web-platform-testsRevert "Find manifest for download by tags instead of commits" Possible but unlikely cause of https://github.com/web-platform-tests/wpt/issues/14207. This reverts commit 3657828ff4243fcc2f3b3fa4e7bed34d646ad565. -- wpt-commits: 5438f72ea0bc2abb3754b2b906a34d497bd8457c wpt-pr: 14208
testing/web-platform/tests/tools/manifest/download.py
--- a/testing/web-platform/tests/tools/manifest/download.py
+++ b/testing/web-platform/tests/tools/manifest/download.py
@@ -28,63 +28,56 @@ def should_download(manifest_path, rebui
         return True
     mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
     if mtime < datetime.now() - rebuild_time:
         return True
     logger.info("Skipping manifest download because existing file is recent")
     return False
 
 
-def merge_pr_tags(repo_root, max_count=50):
+def git_commits(repo_root):
     git = Git.get_func(repo_root)
-    tags = []
-    for line in git("log", "--format=%D", "--max-count=%s" % max_count).split("\n"):
-        for ref in line.split(", "):
-            if ref.startswith("tag: merge_pr_"):
-                tags.append(ref[5:])
-    return tags
+    return [item for item in git("log", "--format=%H", "-n50").split("\n") if item]
 
 
-def github_url(tags):
-    for tag in tags:
-        url = "https://api.github.com/repos/web-platform-tests/wpt/releases/tags/%s" % tag
-        try:
-            resp = urlopen(url)
-        except Exception:
-            logger.warning("Fetching %s failed" % url)
-            continue
+def github_url(commits):
+    try:
+        resp = urlopen("https://api.github.com/repos/web-platform-tests/wpt/releases")
+    except Exception:
+        return None
 
-        if resp.code != 200:
-            logger.warning("Fetching %s failed; got HTTP status %d" % (url, resp.code))
-            continue
+    if resp.code != 200:
+        return None
 
-        try:
-            release = json.load(resp.fp)
-        except ValueError:
-            logger.warning("Response was not valid JSON")
-            return None
+    try:
+        releases = json.load(resp.fp)
+    except ValueError:
+        logger.warning("Response was not valid JSON")
+        return None
 
-        for item in release["assets"]:
-            # Accept both ways of naming the manfest asset, even though
-            # there's no longer a reason to include the commit sha.
-            if item["name"].startswith("MANIFEST-") and item["name"].endswith(".json.gz"):
-                return item["browser_download_url"]
-            elif item["name"] == "MANIFEST.json.gz":
-                return item["browser_download_url"]
-
-    return None
+    fallback = None
+    for release in releases:
+        for commit in commits:
+            for item in release["assets"]:
+                if item["name"] == "MANIFEST-%s.json.gz" % commit:
+                    return item["browser_download_url"]
+                elif item["name"] == "MANIFEST.json.gz" and not fallback:
+                    fallback = item["browser_download_url"]
+    if fallback:
+        logger.info("Can't find a commit-specific manifest so just using the most recent one")
+        return fallback
 
 
-def download_manifest(manifest_path, tags_func, url_func, force=False):
+def download_manifest(manifest_path, commits_func, url_func, force=False):
     if not force and not should_download(manifest_path):
         return False
 
-    tags = tags_func()
+    commits = commits_func()
 
-    url = url_func(tags)
+    url = url_func(commits)
     if not url:
         logger.warning("No generated manifest found")
         return False
 
     logger.info("Downloading manifest from %s" % url)
     try:
         resp = urlopen(url)
     except Exception:
@@ -122,17 +115,17 @@ def create_parser():
         "--tests-root", type=abs_path, default=wpt_root, help="Path to root of tests.")
     parser.add_argument(
         "--force", action="store_true",
         help="Always download, even if the existing manifest is recent")
     return parser
 
 
 def download_from_github(path, tests_root, force=False):
-    return download_manifest(path, lambda: merge_pr_tags(tests_root), github_url,
+    return download_manifest(path, lambda: git_commits(tests_root), github_url,
                              force=force)
 
 
 def run(**kwargs):
     if kwargs["path"] is None:
         path = os.path.join(kwargs["tests_root"], "MANIFEST.json")
     else:
         path = kwargs["path"]