Bug 1303326 - Extend get_apk.py to be able to retrieve the latest nightly & aurora apk files r=jlorenzo
authorSylvestre Ledru <sledru@mozilla.com>
Fri, 16 Sep 2016 14:43:50 +0200
changeset 314482 9b321d41d837235eecd2d5c9a5b40d861a22d55b
parent 314481 28459fc537c796d0ac0ca76887e82af239cf4451
child 314483 cbe6f8892338d106afe6e49ea5425f380935858d
push id81903
push usercbook@mozilla.com
push dateTue, 20 Sep 2016 10:04:26 +0000
treeherdermozilla-inbound@150109898e5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlorenzo
bugs1303326
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1303326 - Extend get_apk.py to be able to retrieve the latest nightly & aurora apk files r=jlorenzo MozReview-Commit-ID: E0urGoFXbFe
testing/mozharness/scripts/get_apk.py
--- a/testing/mozharness/scripts/get_apk.py
+++ b/testing/mozharness/scripts/get_apk.py
@@ -12,17 +12,16 @@ sys.path.insert(1, os.path.dirname(sys.p
 from mozharness.base.script import BaseScript
 from mozharness.base.python import VirtualenvMixin
 from mozharness.base.script import ScriptMixin
 
 
 class GetAPK(BaseScript, VirtualenvMixin):
     all_actions = [
         'create-virtualenv',
-        "test",
         'download-apk'
     ]
 
     default_actions = [
         'create-virtualenv',
         'test'
     ]
 
@@ -32,16 +31,28 @@ class GetAPK(BaseScript, VirtualenvMixin
             "help": "Specify build number (default 1)",
             "default": "1"
         }],
         [["--version"], {
             "dest": "version",
             "help": "Specify version number to download (e.g. 23.0b7)",
             "default": "None"
         }],
+        [["--latest-nightly"], {
+            "dest": "latest_nightly",
+            "help": "Download the latest nightly version",
+            "action": "store_true",
+            "default": False
+        }],
+        [["--latest-aurora"], {
+            "dest": "latest_aurora",
+            "help": "Download the latest aurora version",
+            "action": "store_true",
+            "default": False
+        }],
         [["--arch"], {
             "dest": "arch",
             "help": "Specify which architecture to get the apk for",
             "default": "all"
         }],
         [["--locale"], {
             "dest": "locale",
             "help": "Specify which locale to get the apk for",
@@ -52,25 +63,28 @@ class GetAPK(BaseScript, VirtualenvMixin
             "help": "Use this option to clean the download directory",
             "action": "store_true",
             "default": False
         }]
     ]
 
     arch_values = ["arm", "x86"]
     multi_api_archs = ["arm"]
-    multi_apis = ["api-15"] # v11 has been dropped in fx 46 (1155801)
+    multi_apis = ["api-15"]  # v11 has been dropped in fx 46 (1155801)
     # v9 has been dropped in fx 48 (1220184)
 
     download_dir = "apk-download"
 
     apk_ext = ".apk"
     checksums_ext = ".checksums"
     android_prefix = "android-"
 
+    base_url = "https://ftp.mozilla.org/pub/mobile"
+    json_version_url = "https://product-details.mozilla.org/1.0/firefox_versions.json"
+
     # Cleanup half downloaded files on Ctrl+C
     def signal_handler(self, signal, frame):
         print("You pressed Ctrl+C!")
         self.cleanup()
         sys.exit(1)
 
     def cleanup(self):
         ScriptMixin.rmtree(self, self.download_dir)
@@ -107,26 +121,30 @@ class GetAPK(BaseScript, VirtualenvMixin
 
     # Check the given values are correct
     def check_argument(self):
         if self.config["clean"]:
             self.cleanup()
         if self.config["version"] == "None":
             if self.config["clean"]:
                 sys.exit(0)
-            self.fatal("Version is required")
+        if self.config["version"] != "None" and (self.config["latest_nightly"] or self.config["latest_aurora"]):
+            self.fatal("Cannot set a version and --latest-nightly or --latest-aurora")
 
         if self.config["arch"] not in self.arch_values and not self.config["arch"] == "all":
             error = self.config["arch"] + " is not a valid arch.  " \
                                           "Try one of the following:"+os.linesep
             for arch in self.arch_values:
                 error += arch + os.linesep
             error += "Or don't use the --arch option to download all the archs"
             self.fatal(error)
 
+        if self.config["latest_nightly"] and self.config["latest_aurora"]:
+            self.fatal("Conflicting options. Cannot use --latest-nightly with --latest-aurora")
+
     # Checksum check the APK
     def check_apk(self, apk_file, checksum_file):
         self.info("The checksum for the APK is being checked....")
         checksum = ScriptMixin.read_from_file(self, checksum_file, False)
         checksum = re.sub("\s(.*)", "", checksum.splitlines()[0])
 
         apk_checksum = self.file_sha512sum(apk_file)
 
@@ -134,19 +152,22 @@ class GetAPK(BaseScript, VirtualenvMixin
             self.info("APK checksum check succeeded!")
             self.download_complete(apk_file, checksum_file)
         else:
             ScriptMixin.rmtree(self, self.download_dir)
             self.fatal("Downloading " + apk_file + " failed!")
 
     # Helper functions
     def generate_url(self, version, build, locale, api_suffix, arch_file):
-        return "https://ftp.mozilla.org/pub/mozilla.org/mobile/candidates/" + version + "-candidates/build" + build + \
-               "/" + self.android_prefix + api_suffix + "/" + locale + "/fennec-" + version + "." + locale + \
-               "." + self.android_prefix + arch_file
+        if self.config["latest_nightly"] or self.config["latest_aurora"]:
+            code = "central" if self.config["latest_nightly"] else "aurora"
+            return ("%s/nightly/latest-mozilla-%s-android-%s/fennec-%s.%s.android-%s") % (self.base_url, code, api_suffix, version, locale, arch_file)
+
+        return ("%s/candidates/%s-candidates/build%s/%s%s/%s/fennec-%s.%s.%s%s") % (self.base_url, version, build, self.android_prefix, api_suffix, locale, version, locale, self.android_prefix, arch_file)
+
 
     def get_api_suffix(self, arch):
         if arch in self.multi_api_archs:
             return self.multi_apis
         else:
             return [arch]
 
     def get_arch_file(self, arch):
@@ -173,33 +194,42 @@ class GetAPK(BaseScript, VirtualenvMixin
             if arch in self.multi_api_archs:
                 filename = common_filename + arch_file + "-" + api_suffix
             else:
                 filename = common_filename + arch_file
 
             filename_apk = os.path.join(self.download_dir, filename + self.apk_ext)
             filename_checksums = os.path.join(self.download_dir, filename + self.checksums_ext)
 
+            # Download the APK
             retry_config = {'attempts': 1, 'cleanup': self.download_error}
             ScriptMixin.download_file(self, apk_url, filename_apk, retry_config=retry_config)
 
+            # Download the checksum of the APK
             retry_config = {'attempts': 1, 'cleanup': self.download_error}
             ScriptMixin.download_file(self, checksum_url, filename_checksums, retry_config=retry_config)
 
             self.check_apk(filename_apk, filename_checksums)
 
+    def get_version_name(self):
+        if self.config["latest_nightly"] or self.config["latest_aurora"]:
+            json = self.load_json_url(self.json_version_url)
+            version_code = json['FIREFOX_NIGHTLY'] if self.config["latest_nightly"] else json['FIREFOX_AURORA']
+            return version_code
+        return self.config["version"]
+
     # Download all the archs if none is given
     def download_all(self, version, build, locale):
         for arch in self.arch_values:
             self.download(version, build, arch, locale)
 
     # Download apk initial action
     def download_apk(self):
         self.check_argument()
-        version = self.config["version"]
+        version = self.get_version_name()
         arch = self.config["arch"]
         build = str(self.config["build"])
         locale = self.config["locale"]
 
         self.info("Downloading version " + version + " build #" + build
                   + " for arch " + arch + " (locale " + locale + ")")
         if arch == "all":
             self.download_all(version, build, locale)
@@ -220,18 +250,19 @@ class GetAPK(BaseScript, VirtualenvMixin
 
         self.cleanup()
         if os.path.isdir(self.download_dir):
             self.fatal("cleanup test failed")
 
         url = self.generate_url("43.0", "2", "multi", "x86", "i386")
         correcturl = "https://ftp.mozilla.org/pub/mozilla.org/mobile/candidates/43.0-candidates/build2/"\
                      + self.android_prefix + "x86/multi/fennec-43.0.multi." + self.android_prefix + "i386"
+
         if not url == correcturl:
-            self.fatal("get_url test failed!")
+            self.fatal(("get_url test failed! %s != %s") % (url, correcturl))
 
         if not self.get_api_suffix(self.multi_api_archs[0]) == self.multi_apis:
             self.fatal("get_api_suffix test failed!")
 
         if not self.get_arch_file("x86") == "i386":
             self.fatal("get_arch_file test failed!")
 
         if not self.get_common_file_name("43.0", "multi") == "fennec-43.0.multi." + self.android_prefix: