Bug 821412: Part 4 - Initial automation frontend for B2G update smoketests. r=jgriffin
authorMarshall Culpepper <marshall@mozilla.com>
Wed, 02 Jan 2013 12:41:40 -0600
changeset 126437 067e0c9943e327b8927d5f1f5b9c99ebeddb01d1
parent 126436 80ab5bc94810acfecaf1a1a6a3d7605161d825f6
child 126438 21c0b3fbad22599be180251dfdc767be7dcfaf4d
push id2151
push userlsblakk@mozilla.com
push dateTue, 19 Feb 2013 18:06:57 +0000
treeherdermozilla-beta@4952e88741ec [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgriffin
bugs821412
milestone20.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 821412: Part 4 - Initial automation frontend for B2G update smoketests. r=jgriffin
testing/marionette/update-smoketests/flash-template.sh
testing/marionette/update-smoketests/run-smoketests.py
testing/marionette/update-smoketests/smoketest.py
testing/marionette/update-smoketests/stage-update.py
new file mode 100644
--- /dev/null
+++ b/testing/marionette/update-smoketests/flash-template.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This script is run by the update smoketest frontend
+
+ADB=${ADB:-adb}
+DEVICE=$1
+
+run_adb() {
+    $ADB -s $DEVICE $@
+}
+
+run_adb push %(flash_zip)s %(sdcard)s/_flash.zip
+run_adb shell 'echo -n "--update_package=%(sdcard_recovery)s/_flash.zip" > /cache/recovery/command'
+run_adb reboot recovery
new file mode 100755
--- /dev/null
+++ b/testing/marionette/update-smoketests/run-smoketests.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import os
+import sys
+import tempfile
+
+from smoketest import *
+
+def main():
+    parser = argparse.ArgumentParser(prog="run-smoketests.py")
+    parser.add_argument("--build-dir", required=True,
+        help="directory that contains builds with build ID subdirectories")
+    parser.add_argument("--run-dir", default=None,
+        help="directory where partials and testvars are generated. default: "
+             "create a temp directory")
+    parser.add_argument("--tests", action='append',
+        help="which smoketest(s) to run. by default all tests are run")
+    parser.add_argument("build_ids", nargs="+", metavar="BUILD_ID",
+        help="a list of build IDs to run smoketests against. the IDs will be "
+             "sorted numerically, and partials will be generated from each to "
+             "the last update. this list of partials will be tested along with "
+             "a full update from each build to the last")
+    args = parser.parse_args()
+
+    try:
+        b2g = find_b2g()
+    except EnvironmentError, e:
+        parser.exit(1, "This tool must be run while inside the B2G directory, "
+                       "or B2G_HOME must be set in the environment.")
+
+    if not os.path.exists(args.build_dir):
+        parser.exit(1, "Build dir doesn't exist: %s" % args.build_dir)
+
+    if len(args.build_ids) < 2:
+        parser.exit(1, "This script requires at least two build IDs")
+
+    for test in args.tests:
+        if not os.path.exists(test):
+            parser.exit(1, "Smoketest does not exist: %s" % test)
+
+    try:
+        config = SmokeTestConfig(args.build_dir)
+        runner = SmokeTestRunner(config, b2g, run_dir=args.run_dir)
+        runner.run_smoketests(args.build_ids, args.tests)
+    except KeyError, e:
+        parser.exit(1, "Error: %s" % e)
+    except SmokeTestError, e:
+        parser.exit(1, "Error: %s" % e)
+
+if __name__ == "__main__":
+    main()
new file mode 100644
--- /dev/null
+++ b/testing/marionette/update-smoketests/smoketest.py
@@ -0,0 +1,195 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import threading
+import zipfile
+
+from ConfigParser import ConfigParser
+
+this_dir = os.path.abspath(os.path.dirname(__file__))
+marionette_dir = os.path.dirname(this_dir)
+marionette_client_dir = os.path.join(marionette_dir, 'client', 'marionette')
+
+def find_b2g():
+    sys.path.append(marionette_client_dir)
+    from b2ginstance import B2GInstance
+    return B2GInstance()
+
+class DictObject(dict):
+    def __getattr__(self, item):
+        try:
+            return self.__getitem__(item)
+        except KeyError:
+            raise AttributeError(item)
+
+    def __getitem__(self, item):
+        value = dict.__getitem__(self, item)
+        if isinstance(value, dict):
+            return DictObject(value)
+        return value
+
+class SmokeTestError(Exception):
+    pass
+
+class SmokeTestConfigError(SmokeTestError):
+    def __init__(self, message):
+        SmokeTestError.__init__(self, 'smoketest-config.json: ' + message)
+
+class SmokeTestConfig(DictObject):
+    TOP_LEVEL_REQUIRED = ('devices', 'public_key', 'private_key')
+    DEVICE_REQUIRED    = ('system_fs_type', 'system_location', 'data_fs_type',
+                          'data_location', 'sdcard', 'sdcard_recovery',
+                          'serials')
+
+    def __init__(self, build_dir):
+        self.top_dir = build_dir
+        self.build_data = {}
+        self.flash_template = None
+
+        with open(os.path.join(build_dir, 'smoketest-config.json')) as f:
+            DictObject.__init__(self, json.loads(f.read()))
+
+        for required in self.TOP_LEVEL_REQUIRED:
+            if required not in self:
+                raise SmokeTestConfigError('No "%s" found' % required)
+
+        if len(self.devices) == 0:
+            raise SmokeTestConfigError('No devices found')
+
+        for name, device in self.devices.iteritems():
+            for required in self.DEVICE_REQUIRED:
+                if required not in device:
+                    raise SmokeTestConfigError('No "%s" found in device "%s"' % (required, name))
+
+    def get_build_data(self, device, build_id):
+        if device in self.build_data:
+            if build_id in self.build_data[device]:
+                return self.build_data[device][build_id]
+        else:
+            self.build_data[device] = {}
+
+        build_dir = os.path.join(self.top_dir, device, build_id)
+        flash_zip = os.path.join(build_dir, 'flash.zip')
+        with zipfile.ZipFile(flash_zip) as zip:
+            app_ini = ConfigParser()
+            app_ini.readfp(zip.open('system/b2g/application.ini'))
+            platform_ini = ConfigParser()
+            platform_ini.readfp(zip.open('system/b2g/platform.ini'))
+
+        build_data = self.build_data[device][build_id] = DictObject({
+            'app_version': app_ini.get('App', 'version'),
+            'app_build_id': app_ini.get('App', 'buildid'),
+            'platform_build_id': platform_ini.get('Build', 'buildid'),
+            'platform_milestone': platform_ini.get('Build', 'milestone'),
+            'complete_mar': os.path.join(build_dir, 'complete.mar'),
+            'flash_script': os.path.join(build_dir, 'flash.sh')
+        })
+
+        return build_data
+
+class SmokeTestRunner(object):
+    DEVICE_TIMEOUT = 30
+
+    def __init__(self, config, b2g, run_dir=None):
+        self.config = config
+        self.b2g = b2g
+        self.run_dir = run_dir or tempfile.mkdtemp()
+
+        update_tools = self.b2g.import_update_tools()
+        self.b2g_config = update_tools.B2GConfig()
+
+    def run_b2g_update_test(self, serial, testvars, tests):
+        b2g_update_test = os.path.join(marionette_client_dir,
+                                       'venv_b2g_update_test.sh')
+
+        if not tests:
+            tests = [os.path.join(marionette_client_dir, 'tests',
+                                  'update-tests.ini')]
+
+        args = ['bash', b2g_update_test, sys.executable,
+                '--homedir', self.b2g.homedir,
+                '--address', 'localhost:2828',
+                '--type', 'b2g+smoketest',
+                '--device', serial,
+                '--testvars', testvars]
+        args.extend(tests)
+
+        print ' '.join(args)
+        subprocess.check_call(args)
+
+    def build_testvars(self, device, start_id, finish_id):
+        run_dir = os.path.join(self.run_dir, device, start_id, finish_id)
+        if not os.path.exists(run_dir):
+            os.makedirs(run_dir)
+
+        start_data = self.config.get_build_data(device, start_id)
+        finish_data = self.config.get_build_data(device, finish_id)
+
+        partial_mar = os.path.join(run_dir, 'partial.mar')
+        if not os.path.exists(partial_mar):
+            build_gecko_mar = os.path.join(self.b2g.update_tools,
+                                           'build-gecko-mar.py')
+            subprocess.check_call([build_gecko_mar,
+                                   '--from', start_data.complete_mar,
+                                   '--to', finish_data.complete_mar,
+                                   partial_mar])
+        finish_data['partial_mar'] = partial_mar
+
+        testvars = os.path.join(run_dir, 'testvars.json')
+        if not os.path.exists(testvars):
+            open(testvars, 'w').write(json.dumps({
+                'start': start_data,
+                'finish': finish_data
+            }))
+
+        return testvars
+
+    def wait_for_device(self, device):
+        for serial in self.config.devices[device].serials:
+            proc = subprocess.Popen([self.b2g.adb_path, '-s', serial,
+                                     'wait-for-device'])
+            def wait_for_adb():
+                proc.communicate()
+
+            thread = threading.Thread(target=wait_for_adb)
+            thread.start()
+            thread.join(self.DEVICE_TIMEOUT)
+
+            if thread.isAlive():
+                print >>sys.stderr, '%s device %s is not recognized by ADB, ' \
+                                    'trying next device' % (device, serial)
+                proc.kill()
+                thread.join()
+                continue
+
+            return serial
+        return None
+
+    def run_smoketests_for_device(self, device, start_id, finish_id, tests):
+        testvars = self.build_testvars(device, start_id, finish_id)
+        serial = self.wait_for_device(device)
+        if not serial:
+            raise SmokeTestError('No connected serials for device "%s" could ' \
+                                 'be found' % device)
+
+        try:
+            self.run_b2g_update_test(serial, testvars, tests)
+        except subprocess.CalledProcessError:
+            print >>sys.stderr, 'SMOKETEST-FAIL | START=%s | FINISH=%s | ' \
+                                'DEVICE=%s/%s | %s' % (start_id, finish_id,
+                                                       device, serial, testvars)
+
+    def run_smoketests(self, build_ids, tests):
+        build_ids.sort()
+
+        latest_build_id = build_ids.pop(-1)
+        for build_id in build_ids:
+            for device in self.config.devices:
+                self.run_smoketests_for_device(device, build_id,
+                                               latest_build_id, tests)
new file mode 100755
--- /dev/null
+++ b/testing/marionette/update-smoketests/stage-update.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import json
+import os
+import shutil
+import stat
+import subprocess
+import sys
+
+from ConfigParser import ConfigParser
+from smoketest import *
+
+this_dir = os.path.abspath(os.path.dirname(__file__))
+
+def stage_update(device, stage_dir):
+    config = SmokeTestConfig(stage_dir)
+    if device not in config.devices:
+        raise SmokeTestConfigError('Device "%s" not found' % device)
+
+    device_data = config.devices[device]
+
+    b2g_config = b2g.import_update_tools().B2GConfig()
+    target_out_dir = os.path.join(b2g.homedir, 'out', 'target', 'product', device)
+    app_ini = os.path.join(b2g_config.gecko_objdir, 'dist', 'bin',
+                           'application.ini')
+    gecko_mar = os.path.join(b2g_config.gecko_objdir, 'dist', 'b2g-update',
+                             'b2g-gecko-update.mar')
+
+    build_data = ConfigParser()
+    build_data.read(app_ini)
+    build_id = build_data.get('App', 'buildid')
+    app_version = build_data.get('App', 'version')
+
+    build_dir = os.path.join(config.top_dir, device, build_id)
+    if not os.path.exists(build_dir):
+        os.makedirs(build_dir)
+
+    if not os.path.exists(build_dir):
+        raise SmokeTestError('Couldn\'t create directory: %s' % build_dir)
+
+    build_flash_fota = os.path.join(b2g.update_tools, 'build-flash-fota.py')
+
+    flash_zip = os.path.join(build_dir, 'flash.zip')
+
+    print 'Building flash zip for device %s, version %s, build %s...' % \
+          (device, app_version, build_id)
+
+    subprocess.check_call([build_flash_fota,
+        '--system-dir', os.path.join(target_out_dir, 'system'),
+        '--system-fs-type', device_data.system_fs_type,
+        '--system-location', device_data.system_location,
+        '--data-fs-type', device_data.data_fs_type,
+        '--data-location', device_data.data_location,
+        '--output', flash_zip])
+
+    complete_mar = os.path.join(build_dir, 'complete.mar')
+    shutil.copy(gecko_mar, complete_mar)
+
+    flash_template = open(os.path.join(this_dir, 'flash-template.sh')).read()
+    flash_script = os.path.join(build_dir, 'flash.sh')
+    open(os.path.join(build_dir, 'flash.sh'), 'w').write(flash_template % {
+        'flash_zip': flash_zip,
+        'sdcard': device_data.sdcard,
+        'sdcard_recovery': device_data.sdcard_recovery
+    })
+    os.chmod(flash_script, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+                           stat.S_IRGRP | stat.S_IXGRP |
+                           stat.S_IROTH | stat.S_IXOTH)
+
+def main():
+    parser = argparse.ArgumentParser(prog='stage-update.py')
+    parser.add_argument('device', help='device name for staging')
+    parser.add_argument('stage_dir',
+        help='directory to stage update and flash script for smoketests')
+    args = parser.parse_args()
+
+    try:
+        global b2g
+        b2g = find_b2g()
+    except EnvironmentError, e:
+        parser.exit(1, 'This tool must be run while inside the B2G directory, '
+                       'or B2G_HOME must be set in the environment.')
+
+    try:
+        stage_update(args.device, args.stage_dir)
+    except SmokeTestError, e:
+        parser.exit(1, str(e))
+
+if __name__ == '__main__':
+    main()