Bug 1058286 - Use tooltool.py to download cached artifacts for android builds. r=armenzg
--- a/configs/android/androidarm.py
+++ b/configs/android/androidarm.py
@@ -2,39 +2,36 @@ import os
config = {
"buildbot_json_path": "buildprops.json",
"host_utils_url": "http://talos-remote.pvt.build.mozilla.org/tegra/tegra-host-utils.Linux.742597.zip",
"robocop_package_name": "org.mozilla.roboexample.test",
"device_ip": "127.0.0.1",
"default_sut_port1": "20701",
"default_sut_port2": "20700", # does not prompt for commands
- "tooltool_url": "http://tooltool.pvt.build.mozilla.org/build/sha512",
- "tooltool_cache_path": "/builds/slave/talos-slave/cached",
- "tooltool_cacheable_artifacts": {
- "avd_tar_ball": ("AVDs-armv7a-gingerbread-build-2014-01-23-ubuntu.tar.gz",
- "7140e026b7b747236545dc30e377a959b0bdf91bb4d70efd7f97f92fce12a9196042503124b8df8d30c2d97b7eb5f9df9556afdffa0b5d9625008aead305c32b"),
- },
+ "tooltool_manifest_path": "testing/config/tooltool-manifests/androidarm/releng.manifest",
+ "tooltool_cache": "/builds/tooltool_cache",
+ "tooltool_servers": ["http://runtime-binaries.pvt.build.mozilla.org/tooltool/"],
".avds_dir": "/home/cltbld/.android",
"emulator_process_name": "emulator64-arm",
"emulator_cpu": "cortex-a9",
"exes": {
'adb': '/tools/android-sdk18/platform-tools/adb',
'python': '/tools/buildbot/bin/python',
'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+ 'tooltool.py': "/tools/tooltool.py",
},
"env": {
"DISPLAY": ":0.0",
"PATH": "%(PATH)s:/tools/android-sdk18/tools:/tools/android-sdk18/platform-tools",
"MINIDUMP_SAVEPATH": "%(abs_work_dir)s/../minidumps"
},
"default_actions": [
'clobber',
'read-buildbot-config',
- 'download-cacheable-artifacts',
'setup-avds',
'start-emulators',
'download-and-extract',
'create-virtualenv',
'install',
'run-tests',
'stop-emulators',
],
--- a/configs/android/androidx86.py
+++ b/configs/android/androidx86.py
@@ -2,37 +2,34 @@ import os
config = {
"buildbot_json_path": "buildprops.json",
"host_utils_url": "http://talos-remote.pvt.build.mozilla.org/tegra/tegra-host-utils.Linux.742597.zip",
"robocop_package_name": "org.mozilla.roboexample.test",
"device_ip": "127.0.0.1",
"default_sut_port1": "20701",
"default_sut_port2": "20700", # does not prompt for commands
- "tooltool_url": "http://tooltool.pvt.build.mozilla.org/build/sha512",
- "tooltool_cache_path": "/builds/slave/talos-slave/cached",
- "tooltool_cacheable_artifacts": {
- "avd_tar_ball": ("AVDs-x86-android-4.2_r1-build-2013-11-13-ubuntu.tar.gz",
- "3b2d18eb0194d82c70c5ee17487ccbac309f9b2e9839fe7ca4a27a9a06f6338bb24394476da78559685d99151fccc85fdde03297aa73ee2f7fb3183e11925c4d"),
- },
+ "tooltool_manifest_path": "testing/config/tooltool-manifests/androidx86/releng.manifest",
+ "tooltool_cache": "/builds/tooltool_cache",
+ "tooltool_servers": ["http://runtime-binaries.pvt.build.mozilla.org/tooltool/"],
".avds_dir": "/home/cltbld/.android",
"emulator_process_name": "emulator64-x86",
"exes": {
'adb': '/tools/android-sdk18/platform-tools/adb',
'python': '/tools/buildbot/bin/python',
'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+ 'tooltool.py': "/tools/tooltool.py",
},
"env": {
"DISPLAY": ":0.0",
"PATH": "%(PATH)s:/tools/android-sdk18/tools:/tools/android-sdk18/platform-tools",
},
"default_actions": [
'clobber',
'read-buildbot-config',
- 'download-cacheable-artifacts',
'setup-avds',
'start-emulators',
'download-and-extract',
'create-virtualenv',
'install',
'run-tests',
'stop-emulators',
],
--- a/mozharness/mozilla/tooltool.py
+++ b/mozharness/mozilla/tooltool.py
@@ -10,29 +10,33 @@ TooltoolErrorList = PythonErrorList + [{
}]
class TooltoolMixin(object):
"""Mixin class for handling tooltool manifests.
Requires self.config['tooltool_servers'] to be a list of base urls
"""
def tooltool_fetch(self, manifest, bootstrap_cmd=None,
- output_dir=None, privileged=False):
+ output_dir=None, privileged=False, cache=None):
"""docstring for tooltool_fetch"""
tooltool = self.query_exe('tooltool.py', return_type='list')
cmd = tooltool
# get the tooltools servers from configuration
default_urls = self.config['tooltool_servers']
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(['--url', proxyied_url])
cmd.extend(['fetch', '-m', manifest, '-o'])
+
+ if cache:
+ cmd.extend(['-c', cache])
+
self.retry(
self.run_command,
args=(cmd, ),
kwargs={'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
--- a/scripts/android_emulator_unittest.py
+++ b/scripts/android_emulator_unittest.py
@@ -72,26 +72,24 @@ class AndroidEmulatorTest(BlobUploadMixi
app_name = None
def __init__(self, require_config_file=False):
super(AndroidEmulatorTest, self).__init__(
config_options=self.config_options,
all_actions=['clobber',
'read-buildbot-config',
- 'download-cacheable-artifacts',
'setup-avds',
'start-emulators',
'download-and-extract',
'create-virtualenv',
'install',
'run-tests',
'stop-emulators'],
default_actions=['clobber',
- 'download-cacheable-artifacts',
'start-emulators',
'download-and-extract',
'create-virtualenv',
'install',
'run-tests',
'stop-emulators'],
require_config_file=require_config_file,
config={
@@ -460,48 +458,48 @@ class AndroidEmulatorTest(BlobUploadMixi
"tmp_stdout": tmp_stdout,
"suite_name": suite_name,
"emulator_index": emulator_index
}
##########################################
### Actions for AndroidEmulatorTest ###
##########################################
- def download_cacheable_artifacts(self):
+ def setup_avds(self):
'''
- This will cache every downloadable artifact specified in
- "tooltool_cacheable_artifacts" to "tooltool_cache_path"
+ If tooltool cache mechanism is enabled, the cached version is used by the fetch command
+ If the manifest includes an "unpack" field, tooltool will unpack all compressed archives mentioned in the manifest
'''
c = self.config
- artifacts = c["tooltool_cacheable_artifacts"]
- for artifact_name in artifacts.keys():
- file_name = artifacts[artifact_name][0]
- file_path = os.path.join(c["tooltool_cache_path"], file_name)
- file_shasum = artifacts[artifact_name][1]
- if not os.path.exists(file_path) or self.file_sha512sum(file_path) != file_shasum:
- if os.path.exists(file_path):
- os.remove(file_path)
- # We store files in tooltool as their shasum representation
- file_url = os.path.join(c["tooltool_url"], file_shasum)
- self.download_file(file_url, file_path, c["tooltool_cache_path"])
- if self.file_sha512sum(file_path) != file_shasum:
- return ""
+
+ # TODO
+ # the following code cleans the folder previously (pre bug 1058286) used as cache for tooltool artifacts
+ # it can be removed when the folder has been clobbered on all slaves
+ old_tooltool_cache = "/builds/slave/talos-slave/cached"
+ try:
+ self.rmtree(old_tooltool_cache)
+ self.info("Folder %s is no longer used to cache tooltool artifacts and has been deleted" % old_tooltool_cache)
+ except OSError as e:
+ self.warning("Folder %s has not been clobbered: %s" % (old_tooltool_cache, str(e)))
- def setup_avds(self):
- '''
- We have a tar ball in ToolTool with the pristine templates.
- Let's unpack them every time.
- '''
- c = self.config
+ # FIXME
+ # clobbering and re-unpacking would not be needed if we had a way to check whether
+ # the unpacked content already present match the contents of the tar ball
+
self.rmtree(c[".avds_dir"])
- avd_tar_ball_path = os.path.join(
- c["tooltool_cache_path"],
- c["tooltool_cacheable_artifacts"]["avd_tar_ball"][0])
self.mkdir_p(c[".avds_dir"])
- self.unpack(avd_tar_ball_path, c[".avds_dir"])
+ if self.buildbot_config and 'properties' in self.buildbot_config:
+ url = 'https://hg.mozilla.org/%s/raw-file/%s/%s' % (self.buildbot_config['properties']['repo_path'], self.buildbot_config['properties']['revision'], c["tooltool_manifest_path"])
+ else:
+ self.fatal("properties in self.buildbot_config are required to retrieve tooltool manifest to be used for avds setup")
+ manifest_path = self.download_file(url, file_name='releng.manifest',
+ parent_dir=c[".avds_dir"])
+ if not os.path.exists(manifest_path):
+ self.fatal("Could not retrieve manifest needed to retrieve avds artifacts from %s" % manifest_path)
+ self.tooltool_fetch(manifest_path, output_dir=c[".avds_dir"], cache=c.get("tooltool_cache", None))
def start_emulators(self):
'''
This action starts the emulators and redirects the two SUT ports for each one of them
'''
assert len(self.test_suites) <= len(self.emulators), \
"We can't run more tests that the number of emulators we start"
# We kill compiz because it sometimes prevents us from starting the emulators