Bug 1636910 [wpt PR 23502] - Use tc index for geckodriver downloads, a=testonly
authorJames Graham <james@hoppipolla.co.uk>
Wed, 13 May 2020 10:00:24 +0000
changeset 531179 6d2debab0948c0dd5fc928368589c3c10d4bb374
parent 531178 01914594e45f9fbc9393eea3735fd06d5947f524
child 531180 38475a9adc1e4ce74f26ae7a810ac90d2916920b
push id37435
push userapavel@mozilla.com
push dateWed, 20 May 2020 15:28:23 +0000
treeherdermozilla-central@5415da14ec9a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1636910, 23502
milestone78.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1636910 [wpt PR 23502] - Use tc index for geckodriver downloads, a=testonly Automatic update from web-platform-tests Use tc index for geckodriver downloads Now we have a dedicated build+package task for geckodriver, we can download the binaries from there -- wpt-commits: d8dd60a214e680dee639b7d4be76c2c52f238fd0 wpt-pr: 23502
testing/web-platform/tests/tools/wpt/browser.py
testing/web-platform/tests/tools/wpt/commands.json
testing/web-platform/tests/tools/wptrunner/requirements_firefox.txt
--- a/testing/web-platform/tests/tools/wpt/browser.py
+++ b/testing/web-platform/tests/tools/wpt/browser.py
@@ -13,18 +13,17 @@ from distutils.spawn import find_executa
 from six.moves.urllib.parse import urlsplit
 import requests
 
 from .utils import call, get, untar, unzip
 
 uname = platform.uname()
 
 # the rootUrl for the firefox-ci deployment of Taskcluster
-# (after November 9, https://firefox-ci-tc.services.mozilla.com/)
-FIREFOX_CI_ROOT_URL = 'https://taskcluster.net'
+FIREFOX_CI_ROOT_URL = 'https://firefox-ci-tc.services.mozilla.com'
 
 
 def _get_fileversion(binary, logger=None):
     command = "(Get-Item '%s').VersionInfo.FileVersion" % binary.replace("'", "''")
     try:
         return call("powershell.exe", command).strip()
     except (subprocess.CalledProcessError, OSError):
         if logger is not None:
@@ -44,16 +43,25 @@ def handle_remove_readonly(func, path, e
 def get_ext(filename):
     """Get the extension from a filename with special handling for .tar.foo"""
     name, ext = os.path.splitext(filename)
     if name.endswith(".tar"):
         ext = ".tar%s" % ext
     return ext
 
 
+def get_taskcluster_artifact(index, path):
+    TC_INDEX_BASE = FIREFOX_CI_ROOT_URL + "/api/index/v1/"
+
+    resp = get(TC_INDEX_BASE + "task/%s/artifacts/%s" % (index, path))
+    resp.raise_for_status()
+
+    return resp
+
+
 class Browser(object):
     __metaclass__ = ABCMeta
 
     def __init__(self, logger):
         self.logger = logger
 
     @abstractmethod
     def download(self, dest=None, channel=None, rename=None):
@@ -169,17 +177,17 @@ class Firefox(Browser):
             raise ValueError("Unrecognised release channel: %s" % channel)
 
         if os_key not in os_builds:
             raise ValueError("Unsupported platform: %s %s" % os_key)
 
         url = "https://download.mozilla.org/?product=%s&os=%s&lang=en-US" % (product[channel],
                                                                              os_builds[os_key])
         self.logger.info("Downloading Firefox from %s" % url)
-        resp = requests.get(url)
+        resp = get(url)
 
         filename = None
 
         content_disposition = resp.headers.get('content-disposition')
         if content_disposition:
             filenames = re.findall("filename=(.+)", content_disposition)
             if filenames:
                 filename = filenames[0]
@@ -408,42 +416,42 @@ class Firefox(Browser):
                (version, version, self.platform_string_geckodriver(), format))
         if format == "zip":
             unzip(get(url).raw, dest=dest)
         else:
             untar(get(url).raw, dest=dest)
         return find_executable(os.path.join(dest, "geckodriver"))
 
     def install_geckodriver_nightly(self, dest):
-        import tarfile
-        import mozdownload
         self.logger.info("Attempting to install webdriver from nightly")
-        try:
-            s = mozdownload.DailyScraper(branch="mozilla-central",
-                                         extension="common.tests.tar.gz",
-                                         destination=dest)
-            package_path = s.download()
-        except mozdownload.errors.NotFoundError:
-            return
+
+        platform_bits = ("64" if uname[4] == "x86_64" else
+                         ("32" if self.platform == "win" else ""))
+        tc_platform = "%s%s" % (self.platform, platform_bits)
+
+        archive_ext = ".zip" if uname[0] == "Windows" else ".tar.gz"
+        archive_name = "public/geckodriver%s" % archive_ext
 
         try:
-            exe_suffix = ".exe" if uname[0] == "Windows" else ""
-            with tarfile.open(package_path, "r") as f:
-                try:
-                    member = f.getmember("bin%sgeckodriver%s" % (os.path.sep,
-                                                                 exe_suffix))
-                except KeyError:
-                    return
-                # Remove bin/ from the path.
-                member.name = os.path.basename(member.name)
-                f.extractall(members=[member], path=dest)
-                path = os.path.join(dest, member.name)
-            self.logger.info("Extracted geckodriver to %s" % path)
-        finally:
-            os.unlink(package_path)
+            resp = get_taskcluster_artifact(
+                "gecko.v2.mozilla-central.latest.geckodriver.%s" % tc_platform,
+                archive_name)
+        except Exception:
+            self.logger.info("Geckodriver download failed")
+            return
+
+        if archive_ext == ".zip":
+            unzip(resp.raw, dest)
+        else:
+            untar(resp.raw, dest)
+
+        exe_ext = ".exe" if uname[0] == "Windows" else ""
+        path = os.path.join(dest, "geckodriver%s" % exe_ext)
+
+        self.logger.info("Extracted geckodriver to %s" % path)
 
         return path
 
     def version(self, binary=None, webdriver_binary=None):
         """Retrieve the release version of the installed browser."""
         version_string = call(binary, "--version").strip()
         m = re.match(r"Mozilla Firefox (.*)", version_string)
         if not m:
@@ -456,34 +464,20 @@ class FirefoxAndroid(Browser):
 
     product = "firefox_android"
     requirements = "requirements_firefox.txt"
 
     def download(self, dest=None, channel=None, rename=None):
         if dest is None:
             dest = os.pwd
 
-        if FIREFOX_CI_ROOT_URL == 'https://taskcluster.net':
-            # NOTE: this condition can be removed after November 9, 2019
-            TC_QUEUE_BASE = "https://queue.taskcluster.net/v1/"
-            TC_INDEX_BASE = "https://index.taskcluster.net/v1/"
-        else:
-            TC_QUEUE_BASE = FIREFOX_CI_ROOT_URL + "/api/queue/v1/"
-            TC_INDEX_BASE = FIREFOX_CI_ROOT_URL + "/api/index/v1/"
 
-
-        resp = requests.get(TC_INDEX_BASE +
-                            "task/gecko.v2.mozilla-central.latest.mobile.android-x86_64-opt")
-        resp.raise_for_status()
-        index = resp.json()
-        task_id = index["taskId"]
-
-        resp = requests.get(TC_QUEUE_BASE + "task/%s/artifacts/%s" %
-                            (task_id, "public/build/geckoview-androidTest.apk"))
-        resp.raise_for_status()
+        resp = get_taskcluster_artifact(
+            "gecko.v2.mozilla-central.latest.mobile.android-x86_64-opt",
+            "public/build/geckoview-androidTest.apk")
 
         filename = "geckoview-androidTest.apk"
         if rename:
             filename = "%s%s" % (rename, get_ext(filename)[1])
         apk_path = os.path.join(dest, filename)
 
         with open(apk_path, "wb") as f:
             f.write(resp.content)
--- a/testing/web-platform/tests/tools/wpt/commands.json
+++ b/testing/web-platform/tests/tools/wpt/commands.json
@@ -47,17 +47,16 @@
     "virtualenv": false
   },
   "install": {
     "path": "install.py",
     "script": "run",
     "parser": "get_parser",
     "help": "Install browser components",
     "install": [
-      "mozdownload",
       "mozinstall"
     ]
   },
   "branch-point": {
     "path": "testfiles.py",
     "script": "display_branch_point",
     "parser": null,
     "help": "Print branch point from master",
--- a/testing/web-platform/tests/tools/wptrunner/requirements_firefox.txt
+++ b/testing/web-platform/tests/tools/wptrunner/requirements_firefox.txt
@@ -1,11 +1,10 @@
 marionette_driver==3.0.0
 mozcrash==2.0.0
-mozdownload==1.26.0
 mozinstall==2.0.0
 mozleak==0.2
 moznetwork==1.1.0
 mozprocess==1.0.0
 mozprofile==2.5.0
 mozrunner==7.8.0
 mozversion==2.3.0
 psutil==5.7.0