Bug 1058286 - Use tooltool.py to download cached artifacts for android builds. r=armenzg
☠☠ backed out by 8be7a61b7986 ☠ ☠
authorSimone Bruno <sbruno@mozilla.com>
Fri, 17 Oct 2014 15:07:38 +0200 (2014-10-17)
changeset 3297 edf6b193e3002c758a5b9888f6722dd9f39dd1a1
parent 3295 1b96260e00c5242cbf0d7e6060635547a6d2f24e
child 3298 8be7a61b79862846343084b27bcfe75146ea8b1b
push id2529
push usersbruno@mozilla.com
push dateFri, 17 Oct 2014 13:07:50 +0000 (2014-10-17)
reviewersarmenzg
bugs1058286
Bug 1058286 - Use tooltool.py to download cached artifacts for android builds. r=armenzg
configs/android/androidarm.py
configs/android/androidx86.py
mozharness/mozilla/tooltool.py
scripts/android_emulator_unittest.py
--- a/configs/android/androidarm.py
+++ b/configs/android/androidarm.py
@@ -2,39 +2,36 @@ import os
 
 config = {
     "buildbot_json_path": "buildprops.json",
     "host_utils_url": "http://talos-remote.pvt.build.mozilla.org/tegra/tegra-host-utils.Linux.742597.zip",
     "robocop_package_name": "org.mozilla.roboexample.test",
     "device_ip": "127.0.0.1",
     "default_sut_port1": "20701",
     "default_sut_port2": "20700", # does not prompt for commands
-    "tooltool_url": "http://tooltool.pvt.build.mozilla.org/build/sha512",
-    "tooltool_cache_path": "/builds/slave/talos-slave/cached",
-    "tooltool_cacheable_artifacts": {
-        "avd_tar_ball": ("AVDs-armv7a-gingerbread-build-2014-01-23-ubuntu.tar.gz",
-            "7140e026b7b747236545dc30e377a959b0bdf91bb4d70efd7f97f92fce12a9196042503124b8df8d30c2d97b7eb5f9df9556afdffa0b5d9625008aead305c32b"),
-    },
+    "tooltool_manifest_path": "testing/config/tooltool-manifests/androidarm/releng.manifest",
+    "tooltool_cache": "/builds/tooltool_cache",
+    "tooltool_servers": ["http://runtime-binaries.pvt.build.mozilla.org/tooltool/"],
     ".avds_dir": "/home/cltbld/.android",
     "emulator_process_name": "emulator64-arm",
     "emulator_cpu": "cortex-a9",
     "exes": {
         'adb': '/tools/android-sdk18/platform-tools/adb',
         'python': '/tools/buildbot/bin/python',
         'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+        'tooltool.py': "/tools/tooltool.py",
     },
     "env": {
         "DISPLAY": ":0.0",
         "PATH": "%(PATH)s:/tools/android-sdk18/tools:/tools/android-sdk18/platform-tools",
         "MINIDUMP_SAVEPATH": "%(abs_work_dir)s/../minidumps"
     },
     "default_actions": [
         'clobber',
         'read-buildbot-config',
-        'download-cacheable-artifacts',
         'setup-avds',
         'start-emulators',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-tests',
         'stop-emulators',
     ],
--- a/configs/android/androidx86.py
+++ b/configs/android/androidx86.py
@@ -2,37 +2,34 @@ import os
 
 config = {
     "buildbot_json_path": "buildprops.json",
     "host_utils_url": "http://talos-remote.pvt.build.mozilla.org/tegra/tegra-host-utils.Linux.742597.zip",
     "robocop_package_name": "org.mozilla.roboexample.test",
     "device_ip": "127.0.0.1",
     "default_sut_port1": "20701",
     "default_sut_port2": "20700", # does not prompt for commands
-    "tooltool_url": "http://tooltool.pvt.build.mozilla.org/build/sha512",
-    "tooltool_cache_path": "/builds/slave/talos-slave/cached",
-    "tooltool_cacheable_artifacts": {
-        "avd_tar_ball": ("AVDs-x86-android-4.2_r1-build-2013-11-13-ubuntu.tar.gz",
-            "3b2d18eb0194d82c70c5ee17487ccbac309f9b2e9839fe7ca4a27a9a06f6338bb24394476da78559685d99151fccc85fdde03297aa73ee2f7fb3183e11925c4d"),
-    },
+    "tooltool_manifest_path": "testing/config/tooltool-manifests/androidx86/releng.manifest",
+    "tooltool_cache": "/builds/tooltool_cache",
+    "tooltool_servers": ["http://runtime-binaries.pvt.build.mozilla.org/tooltool/"],
     ".avds_dir": "/home/cltbld/.android",
     "emulator_process_name": "emulator64-x86",
     "exes": {
         'adb': '/tools/android-sdk18/platform-tools/adb',
         'python': '/tools/buildbot/bin/python',
         'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+        'tooltool.py': "/tools/tooltool.py",
     },
     "env": {
         "DISPLAY": ":0.0",
         "PATH": "%(PATH)s:/tools/android-sdk18/tools:/tools/android-sdk18/platform-tools",
     },
     "default_actions": [
         'clobber',
         'read-buildbot-config',
-        'download-cacheable-artifacts',
         'setup-avds',
         'start-emulators',
         'download-and-extract',
         'create-virtualenv',
         'install',
         'run-tests',
         'stop-emulators',
     ],
--- a/mozharness/mozilla/tooltool.py
+++ b/mozharness/mozilla/tooltool.py
@@ -10,29 +10,33 @@ TooltoolErrorList = PythonErrorList + [{
 }]
 
 
 class TooltoolMixin(object):
     """Mixin class for handling tooltool manifests.
     Requires self.config['tooltool_servers'] to be a list of base urls
     """
     def tooltool_fetch(self, manifest, bootstrap_cmd=None,
-                       output_dir=None, privileged=False):
+                       output_dir=None, privileged=False, cache=None):
         """docstring for tooltool_fetch"""
         tooltool = self.query_exe('tooltool.py', return_type='list')
         cmd = tooltool
         # get the tooltools servers from configuration
         default_urls = self.config['tooltool_servers']
         proxxy = Proxxy(self.config, self.log_obj)
         proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
 
         for proxyied_url in proxxy_urls:
             cmd.extend(['--url', proxyied_url])
 
         cmd.extend(['fetch', '-m', manifest, '-o'])
+
+        if cache:
+            cmd.extend(['-c', cache])
+
         self.retry(
             self.run_command,
             args=(cmd, ),
             kwargs={'cwd': output_dir,
                     'error_list': TooltoolErrorList,
                     'privileged': privileged,
                     },
             good_statuses=(0, ),
--- a/scripts/android_emulator_unittest.py
+++ b/scripts/android_emulator_unittest.py
@@ -72,26 +72,24 @@ class AndroidEmulatorTest(BlobUploadMixi
 
     app_name = None
 
     def __init__(self, require_config_file=False):
         super(AndroidEmulatorTest, self).__init__(
             config_options=self.config_options,
             all_actions=['clobber',
                          'read-buildbot-config',
-                         'download-cacheable-artifacts',
                          'setup-avds',
                          'start-emulators',
                          'download-and-extract',
                          'create-virtualenv',
                          'install',
                          'run-tests',
                          'stop-emulators'],
             default_actions=['clobber',
-                             'download-cacheable-artifacts',
                              'start-emulators',
                              'download-and-extract',
                              'create-virtualenv',
                              'install',
                              'run-tests',
                              'stop-emulators'],
             require_config_file=require_config_file,
             config={
@@ -460,48 +458,48 @@ class AndroidEmulatorTest(BlobUploadMixi
             "tmp_stdout": tmp_stdout,
             "suite_name": suite_name,
             "emulator_index": emulator_index
         }
 
     ##########################################
     ### Actions for AndroidEmulatorTest ###
     ##########################################
-    def download_cacheable_artifacts(self):
+    def setup_avds(self):
         '''
-        This will cache every downloadable artifact specified in
-        "tooltool_cacheable_artifacts" to "tooltool_cache_path"
+        If tooltool cache mechanism is enabled, the cached version is used by the fetch command
+        If the manifest includes an "unpack" field, tooltool will unpack all compressed archives mentioned in the manifest
         '''
         c = self.config
-        artifacts = c["tooltool_cacheable_artifacts"]
-        for artifact_name in artifacts.keys():
-            file_name = artifacts[artifact_name][0]
-            file_path = os.path.join(c["tooltool_cache_path"], file_name)
-            file_shasum = artifacts[artifact_name][1]
-            if not os.path.exists(file_path) or self.file_sha512sum(file_path) != file_shasum:
-                if os.path.exists(file_path):
-                    os.remove(file_path)
-                # We store files in tooltool as their shasum representation
-                file_url = os.path.join(c["tooltool_url"], file_shasum)
-                self.download_file(file_url, file_path, c["tooltool_cache_path"])
-                if self.file_sha512sum(file_path) != file_shasum:
-                    return ""
+
+        # TODO
+        # the following code cleans the folder previously (pre bug 1058286) used as cache for tooltool artifacts
+        # it can be removed when the folder has been clobbered on all slaves
+        old_tooltool_cache = "/builds/slave/talos-slave/cached"
+        try:
+            self.rmtree(old_tooltool_cache)
+            self.info("Folder %s is no longer used to cache tooltool artifacts and has been deleted" % old_tooltool_cache)
+        except OSError as e:
+            self.warning("Folder %s has not been clobbered: %s" % (old_tooltool_cache, str(e)))
 
-    def setup_avds(self):
-        '''
-        We have a tar ball in ToolTool with the pristine templates.
-        Let's unpack them every time.
-        '''
-        c = self.config
+        # FIXME
+        # clobbering and re-unpacking would not be needed if we had a way to check whether
+        # the unpacked content already present match the contents of the tar ball
+
         self.rmtree(c[".avds_dir"])
-        avd_tar_ball_path = os.path.join(
-            c["tooltool_cache_path"],
-            c["tooltool_cacheable_artifacts"]["avd_tar_ball"][0])
         self.mkdir_p(c[".avds_dir"])
-        self.unpack(avd_tar_ball_path, c[".avds_dir"])
+        if self.buildbot_config and 'properties' in self.buildbot_config:
+            url = 'https://hg.mozilla.org/%s/raw-file/%s/%s' % (self.buildbot_config['properties']['repo_path'], self.buildbot_config['properties']['revision'], c["tooltool_manifest_path"])
+        else:
+            self.fatal("properties in self.buildbot_config are required to retrieve tooltool manifest to be used for avds setup")
+        manifest_path = self.download_file(url, file_name='releng.manifest',
+                                           parent_dir=c[".avds_dir"])
+        if not os.path.exists(manifest_path):
+            self.fatal("Could not retrieve manifest needed to retrieve avds artifacts from %s" % manifest_path)
+        self.tooltool_fetch(manifest_path, output_dir=c[".avds_dir"], cache=c.get("tooltool_cache", None))
 
     def start_emulators(self):
         '''
         This action starts the emulators and redirects the two SUT ports for each one of them
         '''
         assert len(self.test_suites) <= len(self.emulators), \
             "We can't run more tests that the number of emulators we start"
         # We kill compiz because it sometimes prevents us from starting the emulators