Bug 821412: Part 4 - Initial automation frontend for B2G update smoketests. r=jgriffin
authorMarshall Culpepper <marshall@mozilla.com>
Wed, 02 Jan 2013 12:41:40 -0600
changeset 117352 067e0c9943e327b8927d5f1f5b9c99ebeddb01d1
parent 117351 80ab5bc94810acfecaf1a1a6a3d7605161d825f6
child 117353 21c0b3fbad22599be180251dfdc767be7dcfaf4d
push id1267
push userpastithas@mozilla.com
push dateSat, 05 Jan 2013 09:44:07 +0000
treeherderfx-team@d8ca3e1c469e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgriffin
bugs821412
milestone20.0a1
Bug 821412: Part 4 - Initial automation frontend for B2G update smoketests. r=jgriffin
testing/marionette/update-smoketests/flash-template.sh
testing/marionette/update-smoketests/run-smoketests.py
testing/marionette/update-smoketests/smoketest.py
testing/marionette/update-smoketests/stage-update.py
new file mode 100644
--- /dev/null
+++ b/testing/marionette/update-smoketests/flash-template.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This script is run by the update smoketest frontend
+
+ADB=${ADB:-adb}
+DEVICE=$1
+
+run_adb() {
+    $ADB -s $DEVICE $@
+}
+
+run_adb push %(flash_zip)s %(sdcard)s/_flash.zip
+run_adb shell 'echo -n "--update_package=%(sdcard_recovery)s/_flash.zip" > /cache/recovery/command'
+run_adb reboot recovery
new file mode 100755
--- /dev/null
+++ b/testing/marionette/update-smoketests/run-smoketests.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import os
+import sys
+import tempfile
+
+from smoketest import *
+
+def main():
+    parser = argparse.ArgumentParser(prog="run-smoketests.py")
+    parser.add_argument("--build-dir", required=True,
+        help="directory that contains builds with build ID subdirectories")
+    parser.add_argument("--run-dir", default=None,
+        help="directory where partials and testvars are generated. default: "
+             "create a temp directory")
+    parser.add_argument("--tests", action='append',
+        help="which smoketest(s) to run. by default all tests are run")
+    parser.add_argument("build_ids", nargs="+", metavar="BUILD_ID",
+        help="a list of build IDs to run smoketests against. the IDs will be "
+             "sorted numerically, and partials will be generated from each to "
+             "the last update. this list of partials will be tested along with "
+             "a full update from each build to the last")
+    args = parser.parse_args()
+
+    try:
+        b2g = find_b2g()
+    except EnvironmentError, e:
+        parser.exit(1, "This tool must be run while inside the B2G directory, "
+                       "or B2G_HOME must be set in the environment.")
+
+    if not os.path.exists(args.build_dir):
+        parser.exit(1, "Build dir doesn't exist: %s" % args.build_dir)
+
+    if len(args.build_ids) < 2:
+        parser.exit(1, "This script requires at least two build IDs")
+
+    for test in args.tests:
+        if not os.path.exists(test):
+            parser.exit(1, "Smoketest does not exist: %s" % test)
+
+    try:
+        config = SmokeTestConfig(args.build_dir)
+        runner = SmokeTestRunner(config, b2g, run_dir=args.run_dir)
+        runner.run_smoketests(args.build_ids, args.tests)
+    except KeyError, e:
+        parser.exit(1, "Error: %s" % e)
+    except SmokeTestError, e:
+        parser.exit(1, "Error: %s" % e)
+
+if __name__ == "__main__":
+    main()
new file mode 100644
--- /dev/null
+++ b/testing/marionette/update-smoketests/smoketest.py
@@ -0,0 +1,195 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import threading
+import zipfile
+
+from ConfigParser import ConfigParser
+
+this_dir = os.path.abspath(os.path.dirname(__file__))
+marionette_dir = os.path.dirname(this_dir)
+marionette_client_dir = os.path.join(marionette_dir, 'client', 'marionette')
+
+def find_b2g():
+    sys.path.append(marionette_client_dir)
+    from b2ginstance import B2GInstance
+    return B2GInstance()
+
+class DictObject(dict):
+    def __getattr__(self, item):
+        try:
+            return self.__getitem__(item)
+        except KeyError:
+            raise AttributeError(item)
+
+    def __getitem__(self, item):
+        value = dict.__getitem__(self, item)
+        if isinstance(value, dict):
+            return DictObject(value)
+        return value
+
+class SmokeTestError(Exception):
+    pass
+
+class SmokeTestConfigError(SmokeTestError):
+    def __init__(self, message):
+        SmokeTestError.__init__(self, 'smoketest-config.json: ' + message)
+
+class SmokeTestConfig(DictObject):
+    TOP_LEVEL_REQUIRED = ('devices', 'public_key', 'private_key')
+    DEVICE_REQUIRED    = ('system_fs_type', 'system_location', 'data_fs_type',
+                          'data_location', 'sdcard', 'sdcard_recovery',
+                          'serials')
+
+    def __init__(self, build_dir):
+        self.top_dir = build_dir
+        self.build_data = {}
+        self.flash_template = None
+
+        with open(os.path.join(build_dir, 'smoketest-config.json')) as f:
+            DictObject.__init__(self, json.loads(f.read()))
+
+        for required in self.TOP_LEVEL_REQUIRED:
+            if required not in self:
+                raise SmokeTestConfigError('No "%s" found' % required)
+
+        if len(self.devices) == 0:
+            raise SmokeTestConfigError('No devices found')
+
+        for name, device in self.devices.iteritems():
+            for required in self.DEVICE_REQUIRED:
+                if required not in device:
+                    raise SmokeTestConfigError('No "%s" found in device "%s"' % (required, name))
+
+    def get_build_data(self, device, build_id):
+        if device in self.build_data:
+            if build_id in self.build_data[device]:
+                return self.build_data[device][build_id]
+        else:
+            self.build_data[device] = {}
+
+        build_dir = os.path.join(self.top_dir, device, build_id)
+        flash_zip = os.path.join(build_dir, 'flash.zip')
+        with zipfile.ZipFile(flash_zip) as zip:
+            app_ini = ConfigParser()
+            app_ini.readfp(zip.open('system/b2g/application.ini'))
+            platform_ini = ConfigParser()
+            platform_ini.readfp(zip.open('system/b2g/platform.ini'))
+
+        build_data = self.build_data[device][build_id] = DictObject({
+            'app_version': app_ini.get('App', 'version'),
+            'app_build_id': app_ini.get('App', 'buildid'),
+            'platform_build_id': platform_ini.get('Build', 'buildid'),
+            'platform_milestone': platform_ini.get('Build', 'milestone'),
+            'complete_mar': os.path.join(build_dir, 'complete.mar'),
+            'flash_script': os.path.join(build_dir, 'flash.sh')
+        })
+
+        return build_data
+
+class SmokeTestRunner(object):
+    DEVICE_TIMEOUT = 30
+
+    def __init__(self, config, b2g, run_dir=None):
+        self.config = config
+        self.b2g = b2g
+        self.run_dir = run_dir or tempfile.mkdtemp()
+
+        update_tools = self.b2g.import_update_tools()
+        self.b2g_config = update_tools.B2GConfig()
+
+    def run_b2g_update_test(self, serial, testvars, tests):
+        b2g_update_test = os.path.join(marionette_client_dir,
+                                       'venv_b2g_update_test.sh')
+
+        if not tests:
+            tests = [os.path.join(marionette_client_dir, 'tests',
+                                  'update-tests.ini')]
+
+        args = ['bash', b2g_update_test, sys.executable,
+                '--homedir', self.b2g.homedir,
+                '--address', 'localhost:2828',
+                '--type', 'b2g+smoketest',
+                '--device', serial,
+                '--testvars', testvars]
+        args.extend(tests)
+
+        print ' '.join(args)
+        subprocess.check_call(args)
+
+    def build_testvars(self, device, start_id, finish_id):
+        run_dir = os.path.join(self.run_dir, device, start_id, finish_id)
+        if not os.path.exists(run_dir):
+            os.makedirs(run_dir)
+
+        start_data = self.config.get_build_data(device, start_id)
+        finish_data = self.config.get_build_data(device, finish_id)
+
+        partial_mar = os.path.join(run_dir, 'partial.mar')
+        if not os.path.exists(partial_mar):
+            build_gecko_mar = os.path.join(self.b2g.update_tools,
+                                           'build-gecko-mar.py')
+            subprocess.check_call([build_gecko_mar,
+                                   '--from', start_data.complete_mar,
+                                   '--to', finish_data.complete_mar,
+                                   partial_mar])
+        finish_data['partial_mar'] = partial_mar
+
+        testvars = os.path.join(run_dir, 'testvars.json')
+        if not os.path.exists(testvars):
+            open(testvars, 'w').write(json.dumps({
+                'start': start_data,
+                'finish': finish_data
+            }))
+
+        return testvars
+
+    def wait_for_device(self, device):
+        for serial in self.config.devices[device].serials:
+            proc = subprocess.Popen([self.b2g.adb_path, '-s', serial,
+                                     'wait-for-device'])
+            def wait_for_adb():
+                proc.communicate()
+
+            thread = threading.Thread(target=wait_for_adb)
+            thread.start()
+            thread.join(self.DEVICE_TIMEOUT)
+
+            if thread.isAlive():
+                print >>sys.stderr, '%s device %s is not recognized by ADB, ' \
+                                    'trying next device' % (device, serial)
+                proc.kill()
+                thread.join()
+                continue
+
+            return serial
+        return None
+
+    def run_smoketests_for_device(self, device, start_id, finish_id, tests):
+        testvars = self.build_testvars(device, start_id, finish_id)
+        serial = self.wait_for_device(device)
+        if not serial:
+            raise SmokeTestError('No connected serials for device "%s" could ' \
+                                 'be found' % device)
+
+        try:
+            self.run_b2g_update_test(serial, testvars, tests)
+        except subprocess.CalledProcessError:
+            print >>sys.stderr, 'SMOKETEST-FAIL | START=%s | FINISH=%s | ' \
+                                'DEVICE=%s/%s | %s' % (start_id, finish_id,
+                                                       device, serial, testvars)
+
+    def run_smoketests(self, build_ids, tests):
+        build_ids.sort()
+
+        latest_build_id = build_ids.pop(-1)
+        for build_id in build_ids:
+            for device in self.config.devices:
+                self.run_smoketests_for_device(device, build_id,
+                                               latest_build_id, tests)
new file mode 100755
--- /dev/null
+++ b/testing/marionette/update-smoketests/stage-update.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import json
+import os
+import shutil
+import stat
+import subprocess
+import sys
+
+from ConfigParser import ConfigParser
+from smoketest import *
+
+this_dir = os.path.abspath(os.path.dirname(__file__))
+
+def stage_update(device, stage_dir):
+    config = SmokeTestConfig(stage_dir)
+    if device not in config.devices:
+        raise SmokeTestConfigError('Device "%s" not found' % device)
+
+    device_data = config.devices[device]
+
+    b2g_config = b2g.import_update_tools().B2GConfig()
+    target_out_dir = os.path.join(b2g.homedir, 'out', 'target', 'product', device)
+    app_ini = os.path.join(b2g_config.gecko_objdir, 'dist', 'bin',
+                           'application.ini')
+    gecko_mar = os.path.join(b2g_config.gecko_objdir, 'dist', 'b2g-update',
+                             'b2g-gecko-update.mar')
+
+    build_data = ConfigParser()
+    build_data.read(app_ini)
+    build_id = build_data.get('App', 'buildid')
+    app_version = build_data.get('App', 'version')
+
+    build_dir = os.path.join(config.top_dir, device, build_id)
+    if not os.path.exists(build_dir):
+        os.makedirs(build_dir)
+
+    if not os.path.exists(build_dir):
+        raise SmokeTestError('Couldn\'t create directory: %s' % build_dir)
+
+    build_flash_fota = os.path.join(b2g.update_tools, 'build-flash-fota.py')
+
+    flash_zip = os.path.join(build_dir, 'flash.zip')
+
+    print 'Building flash zip for device %s, version %s, build %s...' % \
+          (device, app_version, build_id)
+
+    subprocess.check_call([build_flash_fota,
+        '--system-dir', os.path.join(target_out_dir, 'system'),
+        '--system-fs-type', device_data.system_fs_type,
+        '--system-location', device_data.system_location,
+        '--data-fs-type', device_data.data_fs_type,
+        '--data-location', device_data.data_location,
+        '--output', flash_zip])
+
+    complete_mar = os.path.join(build_dir, 'complete.mar')
+    shutil.copy(gecko_mar, complete_mar)
+
+    flash_template = open(os.path.join(this_dir, 'flash-template.sh')).read()
+    flash_script = os.path.join(build_dir, 'flash.sh')
+    open(os.path.join(build_dir, 'flash.sh'), 'w').write(flash_template % {
+        'flash_zip': flash_zip,
+        'sdcard': device_data.sdcard,
+        'sdcard_recovery': device_data.sdcard_recovery
+    })
+    os.chmod(flash_script, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+                           stat.S_IRGRP | stat.S_IXGRP |
+                           stat.S_IROTH | stat.S_IXOTH)
+
+def main():
+    parser = argparse.ArgumentParser(prog='stage-update.py')
+    parser.add_argument('device', help='device name for staging')
+    parser.add_argument('stage_dir',
+        help='directory to stage update and flash script for smoketests')
+    args = parser.parse_args()
+
+    try:
+        global b2g
+        b2g = find_b2g()
+    except EnvironmentError, e:
+        parser.exit(1, 'This tool must be run while inside the B2G directory, '
+                       'or B2G_HOME must be set in the environment.')
+
+    try:
+        stage_update(args.device, args.stage_dir)
+    except SmokeTestError, e:
+        parser.exit(1, str(e))
+
+if __name__ == '__main__':
+    main()