Bug 1338177 - Teach fetchlogs.py to download multiple logs in parallel., r=emorley
authorJames Graham <james@hoppipolla.co.uk>
Thu, 02 Feb 2017 14:38:36 +0000
changeset 481540 cef58e7e09619097a3d614a497a81f345469fa03
parent 481539 0b887d9dc591592e8a0cc05bc6d8bb503a6690a5
child 481541 aa534d18c2d25e96243620532d62321ede91ccb9
push id44840
push userbmo:rchien@mozilla.com
push dateFri, 10 Feb 2017 01:03:57 +0000
reviewersemorley
bugs1338177
milestone54.0a1
Bug 1338177 - Teach fetchlogs.py to download multiple logs in parallel., r=emorley MozReview-Commit-ID: 53hwD7YPlYy
testing/web-platform/update/fetchlogs.py
--- a/testing/web-platform/update/fetchlogs.py
+++ b/testing/web-platform/update/fetchlogs.py
@@ -1,16 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import argparse
 import os
 import requests
 import urlparse
+from multiprocessing import Pool
 
 treeherder_base = "https://treeherder.mozilla.org/"
 
 """Simple script for downloading structured logs from treeherder.
 
 For the moment this is specialised to work with web-platform-tests
 logs; in due course it should move somewhere generic and get hooked
 up to mach or similar"""
@@ -89,25 +90,33 @@ def get_structured_logs(branch, commit, 
     jobs_url = urlparse.urljoin(treeherder_base, "/api/project/%s/jobs/" % branch)
     jobs_params = {
         'result_set_id': result_set,
         'count': 2000,
         'exclusion_profile': 'false',
     }
     job_data = fetch_json(jobs_url, params=jobs_params)
 
+    tasks = []
+
     for result in job_data["results"]:
         job_type_name = result["job_type_name"]
         if (job_type_name.startswith("W3C Web Platform") or
             job_type_name.startswith("test-") and "-web-platform-tests-" in job_type_name):
             url = get_blobber_url(branch, result)
             if url:
                 prefix = result["platform"] # platform
-                download(url, prefix, None)
+                tasks.append((url, prefix, None))
 
+    pool = Pool(8)
+    for item in pool.imap_unordered(download_logs, tasks, 1):
+        pass
+
+def download_logs(args):
+    download(*args)
 
 def main():
     parser = create_parser()
     args = parser.parse_args()
 
     get_structured_logs(args.branch, args.commit)
 
 if __name__ == "__main__":