Bug 1519174 - Add debug logging of which taskcluster URLs are being loaded, r=ato
authorJames Graham <james@hoppipolla.co.uk>
Fri, 11 Jan 2019 15:17:34 +0000
changeset 453482 c55708f479737050ecf8fb383001463493b3d012
parent 453481 0c9623b98c22811b1f207406cdf6897e4aeed54c
child 453483 e37abec79d0805045b52b41ca0cb3ee663ebce7f
push id35357
push usernerli@mozilla.com
push dateFri, 11 Jan 2019 21:54:07 +0000
treeherdermozilla-central@0ce024c91511 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato
bugs1519174
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1519174 - Add debug logging of which taskcluster URLs are being loaded, r=ato This should help with diagnosing cases where we fail to download a manifest. Differential Revision: https://phabricator.services.mozilla.com/D16315
testing/web-platform/manifestdownload.py
--- a/testing/web-platform/manifestdownload.py
+++ b/testing/web-platform/manifestdownload.py
@@ -7,16 +7,23 @@ import tarfile
 import requests
 import vcs
 from cStringIO import StringIO
 import logging
 
 HEADERS = {'User-Agent': "wpt manifest download"}
 
 
+def get(logger, url, **kwargs):
+    logger.debug(url)
+    if "headers" not in kwargs:
+        kwargs["headers"] = HEADERS
+    return requests.get(url, **kwargs)
+
+
 def abs_path(path):
     return os.path.abspath(os.path.expanduser(path))
 
 
 def hg_commits(repo_root):
     hg = vcs.Mercurial.get_func(repo_root)
     for item in hg("log", "-fl50", "--template={node}\n", "testing/web-platform/tests",
                    "testing/web-platform/mozilla/tests").splitlines():
@@ -66,43 +73,42 @@ def taskcluster_url(logger, commits):
     for revision in commits:
         req = None
 
         if revision == 40 * "0":
             continue
         try:
             req_headers = HEADERS.copy()
             req_headers.update({'Accept': 'application/json'})
-            req = requests.get(cset_url.format(changeset=revision),
-                               headers=req_headers)
+            req = get(logger, cset_url.format(changeset=revision),
+                      headers=req_headers)
             req.raise_for_status()
         except requests.exceptions.RequestException:
             if req and req.status_code == 404:
                 # The API returns a 404 if it can't find a changeset for the revision.
                 continue
             else:
                 return False
 
         result = req.json()
 
         pushes = result['pushes']
         if not pushes:
             continue
         [cset] = pushes.values()[0]['changesets']
 
         try:
-            req = requests.get(tc_url.format(changeset=cset),
-                               headers=HEADERS)
+            req = get(logger, tc_url.format(changeset=cset))
         except requests.exceptions.RequestException:
             return False
 
         if req.status_code == 200:
             return tc_url.format(changeset=cset) + artifact_path
 
-    logger.info("Can't find a commit-specific manifest so just using the most"
+    logger.info("Can't find a commit-specific manifest so just using the most "
                 "recent one")
 
     return ("https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central."
             "latest.source.manifest-upload" +
             artifact_path)
 
 
 def download_manifest(logger, test_paths, commits_func, url_func, force=False):
@@ -117,17 +123,17 @@ def download_manifest(logger, test_paths
 
     url = url_func(logger, commits)
     if not url:
         logger.warning("No generated manifest found")
         return False
 
     logger.info("Downloading manifest from %s" % url)
     try:
-        req = requests.get(url, headers=HEADERS)
+        req = get(logger, url)
     except Exception:
         logger.warning("Downloading pregenerated manifest failed")
         return False
 
     if req.status_code != 200:
         logger.warning("Downloading pregenerated manifest failed; got"
                         "HTTP status %d" % req.status_code)
         return False