Bug 1230279 - Develop new mozharness script for media-tests for Jenkins; r?maja_zf, whimboo draft
authorSyd Polk <spolk@mozilla.com>
Thu, 03 Dec 2015 15:53:20 -0600
changeset 313980 c7adbbb17ab568fb516e6121913ad08f538c4b76
parent 313910 33d954cc69ff8e0bd22410e3641533e4fac500cf
child 315060 4c8f3e84085ebfe28c90a38dd046c061a06a0943
push id8163
push userspolk@mozilla.com
push dateThu, 03 Dec 2015 21:58:18 +0000
reviewersmaja_zf, whimboo
bugs1230279
milestone45.0a1
Bug 1230279 - Develop new mozharness script for media-tests for Jenkins; r?maja_zf, whimboo The jenkins jobs to call this script will be handled with a changes that need to be made to mozmill-ci, but these changes have to be in place first.
testing/mozharness/configs/mediatests/jenkins_config.py
testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
testing/mozharness/scripts/firefox_media_tests_jenkins.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/mediatests/jenkins_config.py
@@ -0,0 +1,63 @@
+# Default configuration as used by Mozmill CI (Jenkins)
+
+import os
+import sys
+
+import mozharness
+
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    'env': {
+        'PIP_TRUSTED_HOST': 'pypi.pub.build.mozilla.org',
+    },
+
+    # General local variable overwrite
+    'exes': {
+        'gittool.py': [
+            # Bug 1227079 - Python executable eeded to get it executed on Windows
+            sys.executable,
+            os.path.join(external_tools_path, 'gittool.py')
+        ],
+        'hgtool.py': [
+            # Bug 1227079 - Python executable eeded to get it executed on Windows
+            sys.executable,
+            os.path.join(external_tools_path, 'hgtool.py')
+        ],
+    },
+
+    # PIP
+    'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
+    'pip_index': False,
+
+    # mozcrash support
+    'download_minidump_stackwalk': True,
+    'download_symbols': 'ondemand',
+    'download_tooltool': True,
+
+    # Version control information
+    'firefox_media_repo': 'https://github.com/mjzffr/firefox-media-tests.git',
+    'firefox_media_branch': 'master',
+    'firefox_media_rev': 'b11d6c3d7f6af166be314d2ac6673e78c1edb566',
+    'firefox_ui_repo': 'https://github.com/mozilla/firefox-ui-tests.git',
+    'firefox_ui_branch': 'mozilla-central',
+    'firefox_ui_rev': '6d6d57917f85399e903ac69b7e4297091b2d474c',
+
+    # Default test suite
+    'test_suite': 'media-tests',
+
+    'suite_definitions': {
+        'media-tests': {
+            'options': [],
+        },
+        'media-youtube-tests': {
+            'options': [
+                '%(test_manifest)s'
+            ],
+        },
+    },
+}
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
@@ -6,16 +6,17 @@
 # ***** BEGIN LICENSE BLOCK *****
 """firefox_media_tests.py
 
 Author: Maja Frydrychowicz
 """
 import copy
 import os
 import re
+import urlparse
 
 from mozharness.base.log import ERROR, WARNING
 from mozharness.base.script import PreScriptAction
 from mozharness.mozilla.testing.testbase import (TestingMixin,
                                                  testing_config_options)
 from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
 from mozharness.mozilla.vcstools import VCSToolsScript
 
@@ -133,17 +134,19 @@ class FirefoxMediaTestsBase(TestingMixin
      ]
 
     def __init__(self, config_options=None, all_actions=None,
                  default_actions=None, **kwargs):
         self.config_options = media_test_config_options + (config_options or [])
         actions = [
             'clobber',
             'checkout',
+            'download-and-extract',
             'create-virtualenv',
+            'install',
             'run-media-tests',
         ]
         super(FirefoxMediaTestsBase, self).__init__(
             config_options=self.config_options,
             all_actions=all_actions or actions,
             default_actions=default_actions or actions,
             **kwargs
         )
@@ -161,16 +164,28 @@ class FirefoxMediaTestsBase(TestingMixin
                                          'requirements.txt')
         if os.path.isfile(requirements_file):
             self.register_virtualenv_module(requirements=[requirements_file])
         self.register_virtualenv_module(name='firefox-ui-tests',
                                         url=dirs['firefox_ui_dir'])
         self.register_virtualenv_module(name='firefox-media-tests',
                                         url=dirs['firefox_media_dir'])
 
+    def download_and_extract(self):
+        """Overriding method from TestingMixin until firefox-ui-tests are in tree.
+
+        Right now we only care about the installer and symbolds.
+
+        """
+        self._download_installer()
+
+        if self.config.get('download_symbols'):
+            self._download_and_extract_symbols()
+
+
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs()
         dirs = {
             'firefox_media_dir': os.path.join(abs_dirs['abs_work_dir'],
                                               'firefox-media-tests')
         }
@@ -223,27 +238,54 @@ class FirefoxMediaTestsBase(TestingMixin
             cmd += ['--profile', self.profile]
         if self.tests:
             cmd.append(self.tests)
         if self.e10s:
             cmd.append('--e10s')
 
         return cmd
 
+    def query_minidump_stackwalk(self):
+        """We don't have an extracted test package available to get the manifest file.
+
+        So we have to explicitely download the latest version of the manifest from the
+        mozilla-central repository and feed it into the query_minidump_stackwalk() method.
+
+        We can remove this whole method once our tests are part of the tree.
+
+        """
+        print "Made it to query_minidump_stackwalk"
+        manifest_path = None
+
+        if os.environ.get('MINIDUMP_STACKWALK') or self.config.get('download_minidump_stackwalk'):
+            print "We are doing minidump stackwalk!"
+            tooltool_manifest = self.query_minidump_tooltool_manifest()
+            url_base = 'https://hg.mozilla.org/mozilla-central/raw-file/default/testing/'
+
+            dirs = self.query_abs_dirs()
+            manifest_path = os.path.join(dirs['abs_work_dir'], 'releng.manifest')
+            try:
+                self.download_file(urlparse.urljoin(url_base, tooltool_manifest),
+                                   manifest_path)
+            except Exception as e:
+                self.fatal('Download of tooltool manifest file failed: %s' % e.message)
+
+        super(FirefoxMediaTestsBase, self).query_minidump_stackwalk(manifest=manifest_path)
+
+
     def run_media_tests(self):
         cmd = self._query_cmd()
         self.job_result_parser = JobResultParser(
             config=self.config,
             log_obj=self.log_obj,
             error_list=self.error_list
         )
 
         env = self.query_env()
-        if (not os.environ.get('MINIDUMP_STACKWALK') and
-                self.query_minidump_stackwalk()):
+        if self.query_minidump_stackwalk():
             env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
 
         return_code = self.run_command(
             cmd,
             output_timeout=self.test_timeout,
             output_parser=self.job_result_parser,
             env=env
         )
new file mode 100755
--- /dev/null
+++ b/testing/mozharness/scripts/firefox_media_tests_jenkins.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** BEGIN LICENSE BLOCK *****
+"""firefox_media_tests_jenkins.py
+
+Author: Maja Frydrychowicz
+"""
+import copy
+import glob
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.log import ERROR, DEBUG, INFO
+from mozharness.base.script import PreScriptAction, PostScriptAction
+from mozharness.mozilla.blob_upload import (
+    BlobUploadMixin,
+    blobupload_config_options
+)
+from mozharness.mozilla.testing.firefox_media_tests import (
+    FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS
+)
+
+
+class FirefoxMediaTestsJenkins(FirefoxMediaTestsBase):
+
+    def __init__(self):
+        super(FirefoxMediaTestsJenkins, self).__init__(
+            all_actions=['clobber',
+                         'checkout',
+                         'download-and-extract',
+                         'create-virtualenv',
+                         'install',
+                         'run-media-tests',
+                         ],
+        )
+        c = self.config
+        self.installer_url = c.get('installer_url')
+        self.installer_path = c.get('installer_path')
+        self.binary_path = c.get('binary_path')
+        self.test_packages_url = c.get('test_packages_url')
+
+    @PreScriptAction('create-virtualenv')
+    def _pre_create_virtualenv(self, action):
+        dirs = self.query_abs_dirs()
+        requirements = os.path.join(dirs['abs_test_install_dir'],
+                                    'config',
+                                    'marionette_requirements.txt')
+        if os.access(requirements, os.F_OK):
+            self.register_virtualenv_module(requirements=[requirements],
+                                            two_pass=True)
+        super(FirefoxMediaTestsJenkins, self)._pre_create_virtualenv(action)
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        dirs = super(FirefoxMediaTestsJenkins, self).query_abs_dirs()
+        dirs['abs_blob_upload_dir'] = os.path.join(dirs['abs_work_dir'],
+                                                   'blobber_upload_dir')
+        dirs['abs_test_install_dir'] = os.path.join(dirs['abs_work_dir'],
+                                                    'tests')
+        self.abs_dirs = dirs
+        return self.abs_dirs
+
+    def _query_cmd(self):
+        """ Determine how to call firefox-media-tests """
+        cmd = super(FirefoxMediaTestsJenkins, self)._query_cmd()
+        # configure logging
+        dirs = self.query_abs_dirs()
+        blob_upload_dir = dirs.get('abs_blob_upload_dir')
+        cmd += ['--gecko-log', os.path.join(blob_upload_dir,
+                                            'gecko.log')]
+        cmd += ['--log-html', os.path.join(blob_upload_dir,
+                                           'media_tests.html')]
+        cmd += ['--log-mach', os.path.join(blob_upload_dir,
+                                           'media_tests_mach.log')]
+
+        test_suite = self.config.get('test_suite')
+        test_manifest = None if test_suite != 'media-youtube-tests' else \
+            os.path.join(dirs['firefox_media_dir'],
+                         'firefox_media_tests',
+                         'playback', 'youtube', 'manifest.ini')
+        config_fmt_args = {
+            'test_manifest': test_manifest,
+        }
+
+        if test_suite not in self.config["suite_definitions"]:
+            self.fatal("%s is not defined in the config!" % test_suite)
+        for s in self.config["suite_definitions"][test_suite]["options"]:
+            cmd.append(s % config_fmt_args)
+
+        return cmd
+
+    @PostScriptAction('run-media-tests')
+    def _collect_uploads(self, action, success=None):
+        """ Copy extra (log) files to blob upload dir. """
+        dirs = self.query_abs_dirs()
+        log_dir = dirs.get('abs_log_dir')
+        blob_upload_dir = dirs.get('abs_blob_upload_dir')
+        if not log_dir or not blob_upload_dir:
+            return
+        self.mkdir_p(blob_upload_dir)
+        # Move firefox-media-test screenshots into log_dir
+        screenshots_dir = os.path.join(dirs['base_work_dir'],
+                                       'screenshots')
+        log_screenshots_dir = os.path.join(log_dir, 'screenshots')
+        if os.access(log_screenshots_dir, os.F_OK):
+            self.rmtree(log_screenshots_dir)
+        if os.access(screenshots_dir, os.F_OK):
+            self.move(screenshots_dir, log_screenshots_dir)
+
+        # logs to upload: broadest level (info), error, screenshots
+        uploads = glob.glob(os.path.join(log_screenshots_dir, '*'))
+        log_files = self.log_obj.log_files
+        log_level = self.log_obj.log_level
+
+        def append_path(filename, dir=log_dir):
+            if filename:
+                uploads.append(os.path.join(dir, filename))
+
+        append_path(log_files.get(ERROR))
+        # never upload debug logs
+        if log_level == DEBUG:
+            append_path(log_files.get(INFO))
+        else:
+            append_path(log_files.get(log_level))
+        # in case of SimpleFileLogger
+        append_path(log_files.get('default'))
+        for f in uploads:
+            if os.access(f, os.F_OK):
+                dest = os.path.join(blob_upload_dir, os.path.basename(f))
+                self.copyfile(f, dest)
+
+
+if __name__ == '__main__':
+    media_test = FirefoxMediaTestsJenkins()
+    media_test.run_and_exit()