Bug 1197224 - Add mozharness script to run firefox-media-tests in buildbot; r?jgriffin draft
authorMaja Frydrychowicz <mjzffr@gmail.com>
Thu, 10 Sep 2015 15:13:52 -0400
changeset 292055 537d19045119fc300b589663bbcc6b7f016241e6
parent 291313 dd9e40b4695909f1595814c0e79e4d55d73dc283
child 509175 835df98c7ca5a0acee586e745f3a003922d22c17
push id5286
push usermjzffr@gmail.com
push dateThu, 10 Sep 2015 19:29:20 +0000
reviewersjgriffin
bugs1197224
milestone43.0a1
Bug 1197224 - Add mozharness script to run firefox-media-tests in buildbot; r?jgriffin
testing/mozharness/configs/mediatests/buildbot_posix_config.py
testing/mozharness/configs/mediatests/buildbot_windows_config.py
testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
testing/mozharness/scripts/firefox_media_tests_buildbot.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/mediatests/buildbot_posix_config.py
@@ -0,0 +1,49 @@
+import os
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    "virtualenv_path": 'venv',
+    "exes": {
+        'python': '/tools/buildbot/bin/python',
+        'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+        'tooltool.py': "/tools/tooltool.py",
+        'gittool.py': os.path.join(external_tools_path, 'gittool.py'),
+    },
+
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+
+    "buildbot_json_path": "buildprops.json",
+
+    "default_actions": [
+        'clobber',
+        'read-buildbot-config',
+        'checkout',
+        'download-and-extract',
+        'create-virtualenv',
+        'install',
+        'run-media-tests',
+    ],
+    "default_blob_upload_servers": [
+         "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "download_symbols": "ondemand",
+
+    "firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
+    "firefox_media_branch": 'master',
+    "firefox_media_rev": '82c45fba24457b5fe447e967bbcaaec5eb14e3ee',
+    "firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
+    "firefox_ui_branch": 'master',
+    "firefox_ui_rev": '6d6d57917f85399e903ac69b7e4297091b2d474c',
+
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/mediatests/buildbot_windows_config.py
@@ -0,0 +1,57 @@
+import os
+import sys
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    "virtualenv_python_dll": 'c:/mozilla-build/python27/python27.dll',
+    "virtualenv_path": 'venv',
+    "exes": {
+        'python': 'c:/mozilla-build/python27/python',
+        'virtualenv': ['c:/mozilla-build/python27/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
+        'hg': 'c:/mozilla-build/hg/hg',
+        'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(),
+                       '%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()],
+        'tooltool.py': [sys.executable, 'C:/mozilla-build/tooltool.py'],
+        'gittool.py': os.path.join(external_tools_path, 'gittool.py'),
+
+
+    },
+
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+
+    "buildbot_json_path": "buildprops.json",
+
+    "default_actions": [
+        'clobber',
+        'read-buildbot-config',
+        'checkout',
+        'download-and-extract',
+        'create-virtualenv',
+        'install',
+        'run-media-tests',
+    ],
+    "default_blob_upload_servers": [
+         "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
+    "in_tree_config": "config/mozharness/marionette.py",
+    "download_minidump_stackwalk": True,
+    "download_symbols": "ondemand",
+
+    "firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
+    "firefox_media_branch": 'master',
+    "firefox_media_rev": '82c45fba24457b5fe447e967bbcaaec5eb14e3ee',
+    "firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
+    "firefox_ui_branch": 'master',
+    "firefox_ui_rev": '6d6d57917f85399e903ac69b7e4297091b2d474c',
+
+}
new file mode 100755
--- /dev/null
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
@@ -0,0 +1,239 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** BEGIN LICENSE BLOCK *****
+"""firefox_media_tests.py
+
+Author: Maja Frydrychowicz
+"""
+import copy
+import os
+import re
+
+from mozharness.base.log import ERROR, WARNING
+from mozharness.base.script import PreScriptAction
+from mozharness.mozilla.testing.testbase import (TestingMixin,
+                                                 testing_config_options)
+from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
+from mozharness.mozilla.vcstools import VCSToolsScript
+
+BUSTED = 'busted'
+TESTFAILED = 'testfailed'
+UNKNOWN = 'unknown'
+EXCEPTION = 'exception'
+SUCCESS = 'success'
+
+media_test_config_options = [
+    [["--media-urls"],
+     {"action": "store",
+      "dest": "media_urls",
+      "help": "Path to ini file that lists media urls for tests.",
+      }],
+    [["--profile"],
+     {"action": "store",
+      "dest": "profile",
+      "default": None,
+      "help": "Path to FF profile that should be used by Marionette",
+      }],
+    [["--test-timeout"],
+     {"action": "store",
+      "dest": "test_timeout",
+      "default": 10000,
+      "help": ("Number of seconds without output before"
+                "firefox-media-tests is killed."
+                "Set this based on expected time for all media to play."),
+      }],
+    [["--tests"],
+     {"action": "store",
+      "dest": "tests",
+      "default": None,
+      "help": ("Test(s) to run. Path to test_*.py or "
+               "test manifest (*.ini)"),
+      }],
+    [["--e10s"],
+     {"dest": "e10s",
+      "action": "store_true",
+      "default": False,
+      "help": "Enable e10s when running marionette tests."
+      }],
+    [['--firefox-media-repo'], {
+        'dest': 'firefox_media_repo',
+        'default': 'https://github.com/mjzffr/firefox-media-tests.git',
+        'help': 'which firefox_media_tests repo to use',
+    }],
+    [['--firefox-media-branch'], {
+        'dest': 'firefox_media_branch',
+        'default': 'master',
+        'help': 'which branch to use for firefox_media_tests',
+    }],
+    [['--firefox-media-rev'], {
+        'dest': 'firefox_media_rev',
+        'help': 'which firefox_media_tests revision to use',
+    }],
+    [['--firefox-ui-repo'], {
+        'dest': 'firefox_ui_repo',
+        'default': 'https://github.com/mozilla/firefox-ui-tests.git',
+        'help': 'which firefox_ui_tests repo to use',
+    }],
+    [['--firefox-ui-branch'], {
+        'dest': 'firefox_ui_branch',
+        'default': 'master',
+        'help': 'which branch to use for firefox_ui_tests',
+    }],
+    [['--firefox-ui-rev'], {
+        'dest': 'firefox_ui_rev',
+        'help': 'which firefox_ui_tests revision to use',
+    }],
+] + (copy.deepcopy(testing_config_options))
+
+
+class JobResultParser(TestSummaryOutputParserHelper):
+    """ Parses test output to determine overall result."""
+    def __init__(self, **kwargs):
+        super(JobResultParser, self).__init__(**kwargs)
+        self.return_code = 0
+        # External-resource errors that should not count as test failures
+        self.exception_re = re.compile(r'^TEST-UNEXPECTED-ERROR.*'
+                                       r'TimeoutException: Error loading page,'
+                                       r' timed out')
+        self.exceptions = []
+
+    def parse_single_line(self, line):
+        super(JobResultParser, self).parse_single_line(line)
+        if self.exception_re.match(line):
+            self.exceptions.append(line)
+
+    @property
+    def status(self):
+        status = UNKNOWN
+        if self.passed and self.failed == 0:
+            status = SUCCESS
+        elif self.exceptions:
+            status = EXCEPTION
+        elif self.failed:
+            status = TESTFAILED
+        elif self.return_code:
+            status = BUSTED
+        return status
+
+
+class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
+    job_result_parser = None
+
+    error_list = [
+        {'substr': 'FAILED (errors=', 'level': WARNING},
+        {'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
+        {'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR},
+        {'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
+        {'regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH|CRASH|ERROR|FAIL)'''), 'level': ERROR},
+        {'regex': re.compile(r'''(\b\w*Exception)'''), 'level': ERROR},
+        {'regex': re.compile(r'''(\b\w*Error)'''), 'level': ERROR},
+     ]
+
+    def __init__(self, config_options=None, all_actions=None,
+                 default_actions=None, **kwargs):
+        self.config_options = media_test_config_options + (config_options or [])
+        actions = [
+            'clobber',
+            'checkout',
+            'create-virtualenv',
+            'run-media-tests',
+        ]
+        super(FirefoxMediaTestsBase, self).__init__(
+            config_options=self.config_options,
+            all_actions=all_actions or actions,
+            default_actions=default_actions or actions,
+            **kwargs
+        )
+        c = self.config
+        self.media_urls = c.get('media_urls')
+        self.profile = c.get('profile')
+        self.test_timeout = int(c.get('test_timeout'))
+        self.tests = c.get('tests')
+        self.e10s = c.get('e10s')
+
+    @PreScriptAction('create-virtualenv')
+    def _pre_create_virtualenv(self, action):
+        dirs = self.query_abs_dirs()
+        # cwd is $workspace/build
+        self.register_virtualenv_module(name='firefox-ui-tests',
+                                        url=dirs['firefox_ui_dir'],
+                                        method='pip',
+                                        editable='true')
+        self.register_virtualenv_module(name='firefox-media-tests',
+                                        url=dirs['firefox_media_dir'],
+                                        method='pip',
+                                        editable='true')
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs()
+        dirs = {
+            'firefox_media_dir': os.path.join(abs_dirs['abs_work_dir'],
+                                              'firefox-media-tests')
+        }
+        dirs['firefox_ui_dir'] = os.path.join(dirs['firefox_media_dir'],
+                                              'firefox-ui-tests')
+        abs_dirs.update(dirs)
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    @PreScriptAction('checkout')
+    def _pre_checkout(self, action):
+        super(FirefoxMediaTestsBase, self)._pre_checkout(action)
+        c = self.config
+        dirs = self.query_abs_dirs()
+        self.firefox_media_vc = {
+            'branch': c['firefox_media_branch'],
+            'repo': c['firefox_media_repo'],
+            'revision': c['firefox_media_rev'],
+            'dest': dirs['firefox_media_dir'],
+        }
+        self.firefox_ui_vc = {
+            'branch': c['firefox_ui_branch'],
+            'repo': c['firefox_ui_repo'],
+            'revision': c['firefox_ui_rev'],
+            'dest': dirs['firefox_ui_dir']
+        }
+
+    def checkout(self):
+        revision = self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
+        if revision:
+            self.vcs_checkout(vcs='gittool', **self.firefox_ui_vc)
+
+    def _query_cmd(self):
+        """ Determine how to call firefox-media-tests """
+        if not self.binary_path:
+            self.fatal("Binary path could not be determined. "
+                       "Should be set by default during 'install' action.")
+        cmd = ['firefox-media-tests']
+        cmd += ['--binary', self.binary_path]
+        if self.symbols_path:
+            cmd += ['--symbols-path', self.symbols_path]
+        if self.media_urls:
+            cmd += ['--urls', self.media_urls]
+        if self.profile:
+            cmd += ['--profile', self.profile]
+        if self.tests:
+            cmd.append(self.tests)
+        if self.e10s:
+            cmd.append('--e10s')
+        return cmd
+
+    def run_media_tests(self):
+        cmd = self._query_cmd()
+        self.job_result_parser = JobResultParser(
+            config=self.config,
+            log_obj=self.log_obj,
+            error_list=self.error_list
+        )
+        return_code = self.run_command(
+            cmd,
+            output_timeout=self.test_timeout,
+            output_parser=self.job_result_parser
+        )
+        self.job_result_parser.return_code = return_code
+        return self.job_result_parser.status
new file mode 100755
--- /dev/null
+++ b/testing/mozharness/scripts/firefox_media_tests_buildbot.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** BEGIN LICENSE BLOCK *****
+"""firefox_media_tests_buildbot.py
+
+Author: Maja Frydrychowicz
+"""
+import copy
+import glob
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.log import ERROR, DEBUG, INFO
+from mozharness.base.script import PreScriptAction, PostScriptAction
+from mozharness.mozilla.blob_upload import (
+    BlobUploadMixin,
+    blobupload_config_options
+)
+from mozharness.mozilla.buildbot import (
+    TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
+)
+from mozharness.mozilla.testing.firefox_media_tests import (
+    FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS
+)
+
+
+class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
+
+    def __init__(self):
+        config_options = copy.deepcopy(blobupload_config_options)
+        super(FirefoxMediaTestsBuildbot, self).__init__(
+            config_options=config_options,
+            all_actions=['clobber',
+                         'read-buildbot-config',
+                         'checkout',
+                         'download-and-extract',
+                         'create-virtualenv',
+                         'install',
+                         'run-media-tests',
+                         ],
+        )
+        c = self.config
+        self.installer_url = c.get('installer_url')
+        self.installer_path = c.get('installer_path')
+        self.binary_path = c.get('binary_path')
+        self.test_packages_url = c.get('test_packages_url')
+
+    @PreScriptAction('create-virtualenv')
+    def _pre_create_virtualenv(self, action):
+        dirs = self.query_abs_dirs()
+        requirements = os.path.join(dirs['abs_test_install_dir'],
+                                    'config',
+                                    'marionette_requirements.txt')
+        if os.access(requirements, os.F_OK):
+            self.register_virtualenv_module(requirements=[requirements],
+                                            two_pass=True)
+        super(FirefoxMediaTestsBuildbot, self)._pre_create_virtualenv(action)
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
+        dirs['abs_blob_upload_dir'] = os.path.join(dirs['abs_work_dir'],
+                                                   'blobber_upload_dir')
+        dirs['abs_test_install_dir'] = os.path.join(dirs['abs_work_dir'],
+                                                    'tests')
+        self.abs_dirs = dirs
+        return self.abs_dirs
+
+    def _query_cmd(self):
+        """ Determine how to call firefox-media-tests """
+        cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
+        # configure logging
+        dirs = self.query_abs_dirs()
+        blob_upload_dir = dirs.get('abs_blob_upload_dir')
+        cmd += ['--gecko-log', os.path.join(blob_upload_dir,
+                                            'gecko.log')]
+        cmd += ['--log-html', os.path.join(blob_upload_dir,
+                                           'media_tests.html')]
+        cmd += ['--log-mach', os.path.join(blob_upload_dir,
+                                           'media_tests_mach.log')]
+        return cmd
+
+    def run_media_tests(self):
+        status = super(FirefoxMediaTestsBuildbot, self).run_media_tests()
+        if status == SUCCESS:
+            tbpl_status = TBPL_SUCCESS
+        else:
+            tbpl_status = TBPL_FAILURE
+        if status == TESTFAILED:
+            tbpl_status = TBPL_WARNING
+        self.buildbot_status(tbpl_status)
+
+    @PostScriptAction('run-media-tests')
+    def _collect_uploads(self, action, success=None):
+        """ Copy extra (log) files to blob upload dir. """
+        dirs = self.query_abs_dirs()
+        log_dir = dirs.get('abs_log_dir')
+        blob_upload_dir = dirs.get('abs_blob_upload_dir')
+        if not log_dir or not blob_upload_dir:
+            return
+        self.mkdir_p(blob_upload_dir)
+        # Move firefox-media-test screenshots into log_dir
+        screenshots_dir = os.path.join(dirs['base_work_dir'],
+                                       'screenshots')
+        log_screenshots_dir = os.path.join(log_dir, 'screenshots')
+        if os.access(log_screenshots_dir, os.F_OK):
+            self.rmtree(log_screenshots_dir)
+        if os.access(screenshots_dir, os.F_OK):
+            self.move(screenshots_dir, log_screenshots_dir)
+
+        # logs to upload: broadest level (info), error, screenshots
+        uploads = glob.glob(os.path.join(log_screenshots_dir, '*'))
+        log_files = self.log_obj.log_files
+        log_level = self.log_obj.log_level
+
+        def append_path(filename, dir=log_dir):
+            if filename:
+                uploads.append(os.path.join(dir, filename))
+
+        append_path(log_files.get(ERROR))
+        # never upload debug logs
+        if log_level == DEBUG:
+            append_path(log_files.get(INFO))
+        else:
+            append_path(log_files.get(log_level))
+        # in case of SimpleFileLogger
+        append_path(log_files.get('default'))
+        for f in uploads:
+            if os.access(f, os.F_OK):
+                dest = os.path.join(blob_upload_dir, os.path.basename(f))
+                self.copyfile(f, dest)
+
+
+if __name__ == '__main__':
+    media_test = FirefoxMediaTestsBuildbot()
+    media_test.run_and_exit()