Bug 1197224 - Add mozharness script to run firefox-media-tests in buildbot, r=jgriffin, DONTBUILD because NPOTB
authorMaja Frydrychowicz <mjzffr@gmail.com>
Thu, 10 Sep 2015 15:13:52 -0400
changeset 294673 d2457f064ba320b6b32b3b5995d124ec134115fb
parent 294672 2883757f79419f43aaee1923a130352ac3bb9243
child 294674 132aa442af953fab46644e34c60c6782350b703b
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgriffin, DONTBUILD
bugs1197224
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1197224 - Add mozharness script to run firefox-media-tests in buildbot, r=jgriffin, DONTBUILD because NPOTB
testing/mozharness/configs/mediatests/buildbot_posix_config.py
testing/mozharness/configs/mediatests/buildbot_windows_config.py
testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
testing/mozharness/scripts/firefox_media_tests_buildbot.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/mediatests/buildbot_posix_config.py
@@ -0,0 +1,49 @@
+import os
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    "virtualenv_path": 'venv',
+    "exes": {
+        'python': '/tools/buildbot/bin/python',
+        'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+        'tooltool.py': "/tools/tooltool.py",
+        'gittool.py': os.path.join(external_tools_path, 'gittool.py'),
+    },
+
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+
+    "buildbot_json_path": "buildprops.json",
+
+    "default_actions": [
+        'clobber',
+        'read-buildbot-config',
+        'checkout',
+        'download-and-extract',
+        'create-virtualenv',
+        'install',
+        'run-media-tests',
+    ],
+    "default_blob_upload_servers": [
+         "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "download_symbols": "ondemand",
+
+    "firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
+    "firefox_media_branch": 'master',
+    "firefox_media_rev": '82c45fba24457b5fe447e967bbcaaec5eb14e3ee',
+    "firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
+    "firefox_ui_branch": 'master',
+    "firefox_ui_rev": '6d6d57917f85399e903ac69b7e4297091b2d474c',
+
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/mediatests/buildbot_windows_config.py
@@ -0,0 +1,57 @@
+import os
+import sys
+import mozharness
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    "virtualenv_python_dll": 'c:/mozilla-build/python27/python27.dll',
+    "virtualenv_path": 'venv',
+    "exes": {
+        'python': 'c:/mozilla-build/python27/python',
+        'virtualenv': ['c:/mozilla-build/python27/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
+        'hg': 'c:/mozilla-build/hg/hg',
+        'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(),
+                       '%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()],
+        'tooltool.py': [sys.executable, 'C:/mozilla-build/tooltool.py'],
+        'gittool.py': os.path.join(external_tools_path, 'gittool.py'),
+
+
+    },
+
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+
+    "buildbot_json_path": "buildprops.json",
+
+    "default_actions": [
+        'clobber',
+        'read-buildbot-config',
+        'checkout',
+        'download-and-extract',
+        'create-virtualenv',
+        'install',
+        'run-media-tests',
+    ],
+    "default_blob_upload_servers": [
+         "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
+    "in_tree_config": "config/mozharness/marionette.py",
+    "download_minidump_stackwalk": True,
+    "download_symbols": "ondemand",
+
+    "firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
+    "firefox_media_branch": 'master',
+    "firefox_media_rev": '82c45fba24457b5fe447e967bbcaaec5eb14e3ee',
+    "firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
+    "firefox_ui_branch": 'master',
+    "firefox_ui_rev": '6d6d57917f85399e903ac69b7e4297091b2d474c',
+
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
@@ -0,0 +1,239 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** BEGIN LICENSE BLOCK *****
+"""firefox_media_tests.py
+
+Author: Maja Frydrychowicz
+"""
+import copy
+import os
+import re
+
+from mozharness.base.log import ERROR, WARNING
+from mozharness.base.script import PreScriptAction
+from mozharness.mozilla.testing.testbase import (TestingMixin,
+                                                 testing_config_options)
+from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
+from mozharness.mozilla.vcstools import VCSToolsScript
+
+BUSTED = 'busted'
+TESTFAILED = 'testfailed'
+UNKNOWN = 'unknown'
+EXCEPTION = 'exception'
+SUCCESS = 'success'
+
+media_test_config_options = [
+    [["--media-urls"],
+     {"action": "store",
+      "dest": "media_urls",
+      "help": "Path to ini file that lists media urls for tests.",
+      }],
+    [["--profile"],
+     {"action": "store",
+      "dest": "profile",
+      "default": None,
+      "help": "Path to FF profile that should be used by Marionette",
+      }],
+    [["--test-timeout"],
+     {"action": "store",
+      "dest": "test_timeout",
+      "default": 10000,
+      "help": ("Number of seconds without output before"
+                "firefox-media-tests is killed."
+                "Set this based on expected time for all media to play."),
+      }],
+    [["--tests"],
+     {"action": "store",
+      "dest": "tests",
+      "default": None,
+      "help": ("Test(s) to run. Path to test_*.py or "
+               "test manifest (*.ini)"),
+      }],
+    [["--e10s"],
+     {"dest": "e10s",
+      "action": "store_true",
+      "default": False,
+      "help": "Enable e10s when running marionette tests."
+      }],
+    [['--firefox-media-repo'], {
+        'dest': 'firefox_media_repo',
+        'default': 'https://github.com/mjzffr/firefox-media-tests.git',
+        'help': 'which firefox_media_tests repo to use',
+    }],
+    [['--firefox-media-branch'], {
+        'dest': 'firefox_media_branch',
+        'default': 'master',
+        'help': 'which branch to use for firefox_media_tests',
+    }],
+    [['--firefox-media-rev'], {
+        'dest': 'firefox_media_rev',
+        'help': 'which firefox_media_tests revision to use',
+    }],
+    [['--firefox-ui-repo'], {
+        'dest': 'firefox_ui_repo',
+        'default': 'https://github.com/mozilla/firefox-ui-tests.git',
+        'help': 'which firefox_ui_tests repo to use',
+    }],
+    [['--firefox-ui-branch'], {
+        'dest': 'firefox_ui_branch',
+        'default': 'master',
+        'help': 'which branch to use for firefox_ui_tests',
+    }],
+    [['--firefox-ui-rev'], {
+        'dest': 'firefox_ui_rev',
+        'help': 'which firefox_ui_tests revision to use',
+    }],
+] + (copy.deepcopy(testing_config_options))
+
+
+class JobResultParser(TestSummaryOutputParserHelper):
+    """ Parses test output to determine overall result."""
+    def __init__(self, **kwargs):
+        super(JobResultParser, self).__init__(**kwargs)
+        self.return_code = 0
+        # External-resource errors that should not count as test failures
+        self.exception_re = re.compile(r'^TEST-UNEXPECTED-ERROR.*'
+                                       r'TimeoutException: Error loading page,'
+                                       r' timed out')
+        self.exceptions = []
+
+    def parse_single_line(self, line):
+        super(JobResultParser, self).parse_single_line(line)
+        if self.exception_re.match(line):
+            self.exceptions.append(line)
+
+    @property
+    def status(self):
+        status = UNKNOWN
+        if self.passed and self.failed == 0:
+            status = SUCCESS
+        elif self.exceptions:
+            status = EXCEPTION
+        elif self.failed:
+            status = TESTFAILED
+        elif self.return_code:
+            status = BUSTED
+        return status
+
+
+class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
+    job_result_parser = None
+
+    error_list = [
+        {'substr': 'FAILED (errors=', 'level': WARNING},
+        {'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
+        {'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR},
+        {'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
+        {'regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH|CRASH|ERROR|FAIL)'''), 'level': ERROR},
+        {'regex': re.compile(r'''(\b\w*Exception)'''), 'level': ERROR},
+        {'regex': re.compile(r'''(\b\w*Error)'''), 'level': ERROR},
+     ]
+
+    def __init__(self, config_options=None, all_actions=None,
+                 default_actions=None, **kwargs):
+        self.config_options = media_test_config_options + (config_options or [])
+        actions = [
+            'clobber',
+            'checkout',
+            'create-virtualenv',
+            'run-media-tests',
+        ]
+        super(FirefoxMediaTestsBase, self).__init__(
+            config_options=self.config_options,
+            all_actions=all_actions or actions,
+            default_actions=default_actions or actions,
+            **kwargs
+        )
+        c = self.config
+        self.media_urls = c.get('media_urls')
+        self.profile = c.get('profile')
+        self.test_timeout = int(c.get('test_timeout'))
+        self.tests = c.get('tests')
+        self.e10s = c.get('e10s')
+
+    @PreScriptAction('create-virtualenv')
+    def _pre_create_virtualenv(self, action):
+        dirs = self.query_abs_dirs()
+        # cwd is $workspace/build
+        self.register_virtualenv_module(name='firefox-ui-tests',
+                                        url=dirs['firefox_ui_dir'],
+                                        method='pip',
+                                        editable='true')
+        self.register_virtualenv_module(name='firefox-media-tests',
+                                        url=dirs['firefox_media_dir'],
+                                        method='pip',
+                                        editable='true')
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs()
+        dirs = {
+            'firefox_media_dir': os.path.join(abs_dirs['abs_work_dir'],
+                                              'firefox-media-tests')
+        }
+        dirs['firefox_ui_dir'] = os.path.join(dirs['firefox_media_dir'],
+                                              'firefox-ui-tests')
+        abs_dirs.update(dirs)
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    @PreScriptAction('checkout')
+    def _pre_checkout(self, action):
+        super(FirefoxMediaTestsBase, self)._pre_checkout(action)
+        c = self.config
+        dirs = self.query_abs_dirs()
+        self.firefox_media_vc = {
+            'branch': c['firefox_media_branch'],
+            'repo': c['firefox_media_repo'],
+            'revision': c['firefox_media_rev'],
+            'dest': dirs['firefox_media_dir'],
+        }
+        self.firefox_ui_vc = {
+            'branch': c['firefox_ui_branch'],
+            'repo': c['firefox_ui_repo'],
+            'revision': c['firefox_ui_rev'],
+            'dest': dirs['firefox_ui_dir']
+        }
+
+    def checkout(self):
+        revision = self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
+        if revision:
+            self.vcs_checkout(vcs='gittool', **self.firefox_ui_vc)
+
+    def _query_cmd(self):
+        """ Determine how to call firefox-media-tests """
+        if not self.binary_path:
+            self.fatal("Binary path could not be determined. "
+                       "Should be set by default during 'install' action.")
+        cmd = ['firefox-media-tests']
+        cmd += ['--binary', self.binary_path]
+        if self.symbols_path:
+            cmd += ['--symbols-path', self.symbols_path]
+        if self.media_urls:
+            cmd += ['--urls', self.media_urls]
+        if self.profile:
+            cmd += ['--profile', self.profile]
+        if self.tests:
+            cmd.append(self.tests)
+        if self.e10s:
+            cmd.append('--e10s')
+        return cmd
+
+    def run_media_tests(self):
+        cmd = self._query_cmd()
+        self.job_result_parser = JobResultParser(
+            config=self.config,
+            log_obj=self.log_obj,
+            error_list=self.error_list
+        )
+        return_code = self.run_command(
+            cmd,
+            output_timeout=self.test_timeout,
+            output_parser=self.job_result_parser
+        )
+        self.job_result_parser.return_code = return_code
+        return self.job_result_parser.status
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/firefox_media_tests_buildbot.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** BEGIN LICENSE BLOCK *****
+"""firefox_media_tests_buildbot.py
+
+Author: Maja Frydrychowicz
+"""
+import copy
+import glob
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.log import ERROR, DEBUG, INFO
+from mozharness.base.script import PreScriptAction, PostScriptAction
+from mozharness.mozilla.blob_upload import (
+    BlobUploadMixin,
+    blobupload_config_options
+)
+from mozharness.mozilla.buildbot import (
+    TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
+)
+from mozharness.mozilla.testing.firefox_media_tests import (
+    FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS
+)
+
+
+class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
+
+    def __init__(self):
+        config_options = copy.deepcopy(blobupload_config_options)
+        super(FirefoxMediaTestsBuildbot, self).__init__(
+            config_options=config_options,
+            all_actions=['clobber',
+                         'read-buildbot-config',
+                         'checkout',
+                         'download-and-extract',
+                         'create-virtualenv',
+                         'install',
+                         'run-media-tests',
+                         ],
+        )
+        c = self.config
+        self.installer_url = c.get('installer_url')
+        self.installer_path = c.get('installer_path')
+        self.binary_path = c.get('binary_path')
+        self.test_packages_url = c.get('test_packages_url')
+
+    @PreScriptAction('create-virtualenv')
+    def _pre_create_virtualenv(self, action):
+        dirs = self.query_abs_dirs()
+        requirements = os.path.join(dirs['abs_test_install_dir'],
+                                    'config',
+                                    'marionette_requirements.txt')
+        if os.access(requirements, os.F_OK):
+            self.register_virtualenv_module(requirements=[requirements],
+                                            two_pass=True)
+        super(FirefoxMediaTestsBuildbot, self)._pre_create_virtualenv(action)
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
+        dirs['abs_blob_upload_dir'] = os.path.join(dirs['abs_work_dir'],
+                                                   'blobber_upload_dir')
+        dirs['abs_test_install_dir'] = os.path.join(dirs['abs_work_dir'],
+                                                    'tests')
+        self.abs_dirs = dirs
+        return self.abs_dirs
+
+    def _query_cmd(self):
+        """ Determine how to call firefox-media-tests """
+        cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
+        # configure logging
+        dirs = self.query_abs_dirs()
+        blob_upload_dir = dirs.get('abs_blob_upload_dir')
+        cmd += ['--gecko-log', os.path.join(blob_upload_dir,
+                                            'gecko.log')]
+        cmd += ['--log-html', os.path.join(blob_upload_dir,
+                                           'media_tests.html')]
+        cmd += ['--log-mach', os.path.join(blob_upload_dir,
+                                           'media_tests_mach.log')]
+        return cmd
+
+    def run_media_tests(self):
+        status = super(FirefoxMediaTestsBuildbot, self).run_media_tests()
+        if status == SUCCESS:
+            tbpl_status = TBPL_SUCCESS
+        else:
+            tbpl_status = TBPL_FAILURE
+        if status == TESTFAILED:
+            tbpl_status = TBPL_WARNING
+        self.buildbot_status(tbpl_status)
+
+    @PostScriptAction('run-media-tests')
+    def _collect_uploads(self, action, success=None):
+        """ Copy extra (log) files to blob upload dir. """
+        dirs = self.query_abs_dirs()
+        log_dir = dirs.get('abs_log_dir')
+        blob_upload_dir = dirs.get('abs_blob_upload_dir')
+        if not log_dir or not blob_upload_dir:
+            return
+        self.mkdir_p(blob_upload_dir)
+        # Move firefox-media-test screenshots into log_dir
+        screenshots_dir = os.path.join(dirs['base_work_dir'],
+                                       'screenshots')
+        log_screenshots_dir = os.path.join(log_dir, 'screenshots')
+        if os.access(log_screenshots_dir, os.F_OK):
+            self.rmtree(log_screenshots_dir)
+        if os.access(screenshots_dir, os.F_OK):
+            self.move(screenshots_dir, log_screenshots_dir)
+
+        # logs to upload: broadest level (info), error, screenshots
+        uploads = glob.glob(os.path.join(log_screenshots_dir, '*'))
+        log_files = self.log_obj.log_files
+        log_level = self.log_obj.log_level
+
+        def append_path(filename, dir=log_dir):
+            if filename:
+                uploads.append(os.path.join(dir, filename))
+
+        append_path(log_files.get(ERROR))
+        # never upload debug logs
+        if log_level == DEBUG:
+            append_path(log_files.get(INFO))
+        else:
+            append_path(log_files.get(log_level))
+        # in case of SimpleFileLogger
+        append_path(log_files.get('default'))
+        for f in uploads:
+            if os.access(f, os.F_OK):
+                dest = os.path.join(blob_upload_dir, os.path.basename(f))
+                self.copyfile(f, dest)
+
+
+if __name__ == '__main__':
+    media_test = FirefoxMediaTestsBuildbot()
+    media_test.run_and_exit()