Bug 1230279 - Develop new mozharness script for media-tests for Jenkins; r=maja_zf r=whimboo
authorSyd Polk <spolk@mozilla.com>
Mon, 11 Jan 2016 15:52:25 -0600
changeset 279940 6f207cecddefbe27efdeb6adece7ba2e025317bd
parent 279939 a7db157c7baa9f02cb23bc6b9e0d227d50638dd9
child 279941 3a8ebac26ca580cf5235a37a2ff80c13ac7b67e6
push id70273
push usercbook@mozilla.com
push dateThu, 14 Jan 2016 09:44:28 +0000
treeherdermozilla-inbound@6f207cecddef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmaja_zf, whimboo
bugs1230279
milestone46.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1230279 - Develop new mozharness script for media-tests for Jenkins; r=maja_zf r=whimboo
testing/mozharness/configs/mediatests/buildbot_posix_config.py
testing/mozharness/configs/mediatests/buildbot_windows_config.py
testing/mozharness/configs/mediatests/jenkins_config.py
testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
testing/mozharness/scripts/firefox_media_tests_buildbot.py
testing/mozharness/scripts/firefox_media_tests_jenkins.py
--- a/testing/mozharness/configs/mediatests/buildbot_posix_config.py
+++ b/testing/mozharness/configs/mediatests/buildbot_posix_config.py
@@ -36,20 +36,17 @@ config = {
          "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
     "download_minidump_stackwalk": True,
     "download_symbols": "ondemand",
 
     "firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
     "firefox_media_branch": 'master',
-    "firefox_media_rev": '49b500b30b80372a6c678ec7d0a2b074844f5e84',
-    "firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
-    "firefox_ui_branch": 'mozilla-central',
-    "firefox_ui_rev": '32be49d74e1d10c6bf087235b1d6753c1b840bc4',
+    "firefox_media_rev": '0830e972e4b95fef3507207fc6bce028da27f2d3',
 
     "suite_definitions": {
         "media-tests": {
             "options": [],
         },
         "media-youtube-tests": {
             "options": [
                 "%(test_manifest)s"
--- a/testing/mozharness/configs/mediatests/buildbot_windows_config.py
+++ b/testing/mozharness/configs/mediatests/buildbot_windows_config.py
@@ -47,20 +47,17 @@ config = {
     ],
     "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
     "in_tree_config": "config/mozharness/marionette.py",
     "download_minidump_stackwalk": True,
     "download_symbols": "ondemand",
 
     "firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
     "firefox_media_branch": 'master',
-    "firefox_media_rev": '49b500b30b80372a6c678ec7d0a2b074844f5e84',
-    "firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
-    "firefox_ui_branch": 'mozilla-central',
-    "firefox_ui_rev": '32be49d74e1d10c6bf087235b1d6753c1b840bc4',
+    "firefox_media_rev": '0830e972e4b95fef3507207fc6bce028da27f2d3',
 
     "suite_definitions": {
         "media-tests": {
             "options": [],
         },
         "media-youtube-tests": {
             "options": [
                 "%(test_manifest)s"
new file mode 100755
--- /dev/null
+++ b/testing/mozharness/configs/mediatests/jenkins_config.py
@@ -0,0 +1,72 @@
+# Default configuration as used by Mozmill CI (Jenkins)
+
+import os
+import platform
+import sys
+
+import mozharness
+
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    'env': {
+        'PIP_TRUSTED_HOST': 'pypi.pub.build.mozilla.org',
+    },
+
+    # PIP
+    'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
+    'pip_index': False,
+
+    # mozcrash support
+    'download_minidump_stackwalk': True,
+    'download_symbols': 'ondemand',
+    'download_tooltool': True,
+
+    # Version control information
+    'firefox_media_repo': 'https://github.com/mjzffr/firefox-media-tests.git',
+    'firefox_media_branch': 'master',
+
+    # Default test suite
+    'test_suite': 'media-tests',
+
+    'suite_definitions': {
+        'media-tests': {
+            'options': [],
+        },
+        'media-youtube-tests': {
+            'options': [
+                '%(test_manifest)s'
+            ],
+        },
+    },
+
+    'default_actions': [
+        'clobber',
+        'checkout',
+        'download-and-extract',
+        'create-virtualenv',
+        'install',
+        'run-media-tests',
+    ],
+
+}
+
+# General local variable overwrite
+# Bug 1227079 - Python executable eeded to get it executed on Windows
+if platform.system() == 'windows':
+    gittool = [
+        sys.executable,
+        os.path.join(external_tools_path, 'gittool.py')
+    ]
+else:
+    gittool = os.path.join(external_tools_path, 'gittool.py')
+
+exes = {
+    'gittool.py' : gittool,
+}
+
+config['exes'] = exes
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
@@ -1,21 +1,20 @@
 #!/usr/bin/env python
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** BEGIN LICENSE BLOCK *****
-"""firefox_media_tests.py
 
-Author: Maja Frydrychowicz
-"""
 import copy
+import glob
 import os
 import re
+import urlparse
 
 from mozharness.base.log import ERROR, WARNING
 from mozharness.base.script import PreScriptAction
 from mozharness.mozilla.testing.testbase import (TestingMixin,
                                                  testing_config_options)
 from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
 from mozharness.mozilla.vcstools import VCSToolsScript
 
@@ -67,33 +66,24 @@ media_test_config_options = [
         'dest': 'firefox_media_branch',
         'default': 'master',
         'help': 'which branch to use for firefox_media_tests',
     }],
     [['--firefox-media-rev'], {
         'dest': 'firefox_media_rev',
         'help': 'which firefox_media_tests revision to use',
     }],
-    [['--firefox-ui-repo'], {
-        'dest': 'firefox_ui_repo',
-        'default': 'https://github.com/mozilla/firefox-ui-tests.git',
-        'help': 'which firefox_ui_tests repo to use',
-    }],
-    [['--firefox-ui-branch'], {
-        'dest': 'firefox_ui_branch',
-        'default': 'master',
-        'help': 'which branch to use for firefox_ui_tests',
-    }],
-    [['--firefox-ui-rev'], {
-        'dest': 'firefox_ui_rev',
-        'help': 'which firefox_ui_tests revision to use',
-    }],
+    [["--suite"],
+     {"action": "store",
+      "dest": "test_suite",
+      "default": "media-tests",
+      "help": "suite name",
+      }],
 ] + (copy.deepcopy(testing_config_options))
 
-
 class JobResultParser(TestSummaryOutputParserHelper):
     """ Parses test output to determine overall result."""
     def __init__(self, **kwargs):
         super(JobResultParser, self).__init__(**kwargs)
         self.return_code = 0
         # External-resource errors that should not count as test failures
         self.exception_re = re.compile(r'^TEST-UNEXPECTED-ERROR.*'
                                        r'TimeoutException: Error loading page,'
@@ -133,80 +123,97 @@ class FirefoxMediaTestsBase(TestingMixin
      ]
 
     def __init__(self, config_options=None, all_actions=None,
                  default_actions=None, **kwargs):
         self.config_options = media_test_config_options + (config_options or [])
         actions = [
             'clobber',
             'checkout',
+            'download-and-extract',
             'create-virtualenv',
+            'install',
             'run-media-tests',
         ]
         super(FirefoxMediaTestsBase, self).__init__(
             config_options=self.config_options,
             all_actions=all_actions or actions,
             default_actions=default_actions or actions,
             **kwargs
         )
         c = self.config
+
         self.media_urls = c.get('media_urls')
         self.profile = c.get('profile')
         self.test_timeout = int(c.get('test_timeout'))
         self.tests = c.get('tests')
         self.e10s = c.get('e10s')
+        self.installer_url = c.get('installer_url')
+        self.installer_path = c.get('installer_path')
+        self.binary_path = c.get('binary_path')
+        self.test_packages_url = c.get('test_packages_url')
+        self.test_url = c.get('test_url')
 
     @PreScriptAction('create-virtualenv')
     def _pre_create_virtualenv(self, action):
         dirs = self.query_abs_dirs()
-        requirements_file = os.path.join(dirs['firefox_media_dir'],
+        marionette_requirements = os.path.join(dirs['abs_test_install_dir'],
+                                    'config',
+                                    'marionette_requirements.txt')
+        if os.access(marionette_requirements, os.F_OK):
+            self.register_virtualenv_module(requirements=[marionette_requirements],
+                                            two_pass=True)
+
+        media_tests_requirements = os.path.join(dirs['firefox_media_dir'],
                                          'requirements.txt')
-        if os.path.isfile(requirements_file):
-            self.register_virtualenv_module(requirements=[requirements_file])
-        self.register_virtualenv_module(name='firefox-ui-tests',
-                                        url=dirs['firefox_ui_dir'])
-        self.register_virtualenv_module(name='firefox-media-tests',
-                                        url=dirs['firefox_media_dir'])
+
+        if os.access(media_tests_requirements, os.F_OK):
+            self.register_virtualenv_module(requirements=[media_tests_requirements],
+                                            two_pass=True)
+
+    def download_and_extract(self):
+        """Overriding method from TestingMixin until firefox-media-tests are in tree.
+
+        Right now we only care about the installer and symbolds.
+
+        """
+        self._download_installer()
+
+        if self.config.get('download_symbols'):
+            self._download_and_extract_symbols()
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs()
         dirs = {
             'firefox_media_dir': os.path.join(abs_dirs['abs_work_dir'],
                                               'firefox-media-tests')
         }
-        dirs['firefox_ui_dir'] = os.path.join(dirs['firefox_media_dir'],
-                                              'firefox-ui-tests')
+        dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'],
+                                                    'tests')
         abs_dirs.update(dirs)
         self.abs_dirs = abs_dirs
         return self.abs_dirs
 
     @PreScriptAction('checkout')
     def _pre_checkout(self, action):
         super(FirefoxMediaTestsBase, self)._pre_checkout(action)
         c = self.config
         dirs = self.query_abs_dirs()
         self.firefox_media_vc = {
             'branch': c['firefox_media_branch'],
             'repo': c['firefox_media_repo'],
-            'revision': c['firefox_media_rev'],
             'dest': dirs['firefox_media_dir'],
         }
-        self.firefox_ui_vc = {
-            'branch': c['firefox_ui_branch'],
-            'repo': c['firefox_ui_repo'],
-            'revision': c['firefox_ui_rev'],
-            'dest': dirs['firefox_ui_dir']
-        }
+        if 'firefox-media-rev' in c:
+            self.firefox_media_vc['revision'] = c['firefox_media_rev']
 
     def checkout(self):
-        revision = self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
-        if revision:
-            self.vcs_checkout(vcs='gittool', **self.firefox_ui_vc)
+        self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
 
     def _query_cmd(self):
         """ Determine how to call firefox-media-tests """
         if not self.binary_path:
             self.fatal("Binary path could not be determined. "
                        "Should be set by default during 'install' action.")
         dirs = self.query_abs_dirs()
         venv_python_path = self.query_python_path()
@@ -221,29 +228,68 @@ class FirefoxMediaTestsBase(TestingMixin
             cmd += ['--urls', self.media_urls]
         if self.profile:
             cmd += ['--profile', self.profile]
         if self.tests:
             cmd.append(self.tests)
         if self.e10s:
             cmd.append('--e10s')
 
+        test_suite = self.config.get('test_suite')
+        if test_suite not in self.config["suite_definitions"]:
+            self.fatal("%s is not defined in the config!" % test_suite)
+
+        test_manifest = None if test_suite != 'media-youtube-tests' else \
+            os.path.join(dirs['firefox_media_dir'],
+                         'firefox_media_tests',
+                         'playback', 'youtube', 'manifest.ini')
+        config_fmt_args = {
+            'test_manifest': test_manifest,
+        }
+
+        for s in self.config["suite_definitions"][test_suite]["options"]:
+            cmd.append(s % config_fmt_args)
+
         return cmd
 
+    def query_minidump_stackwalk(self):
+        """We don't have an extracted test package available to get the manifest file.
+
+        So we have to explicitely download the latest version of the manifest from the
+        mozilla-central repository and feed it into the query_minidump_stackwalk() method.
+
+        We can remove this whole method once our tests are part of the tree.
+
+        """
+        manifest_path = None
+
+        if os.environ.get('MINIDUMP_STACKWALK') or self.config.get('download_minidump_stackwalk'):
+            tooltool_manifest = self.query_minidump_tooltool_manifest()
+            url_base = 'https://hg.mozilla.org/mozilla-central/raw-file/default/testing/'
+
+            dirs = self.query_abs_dirs()
+            manifest_path = os.path.join(dirs['abs_work_dir'], 'releng.manifest')
+            try:
+                self.download_file(urlparse.urljoin(url_base, tooltool_manifest),
+                                   manifest_path)
+            except Exception as e:
+                self.fatal('Download of tooltool manifest file failed: %s' % e.message)
+
+        return super(FirefoxMediaTestsBase, self).query_minidump_stackwalk(manifest=manifest_path)
+
     def run_media_tests(self):
         cmd = self._query_cmd()
         self.job_result_parser = JobResultParser(
             config=self.config,
             log_obj=self.log_obj,
             error_list=self.error_list
         )
 
         env = self.query_env()
-        if (not os.environ.get('MINIDUMP_STACKWALK') and
-                self.query_minidump_stackwalk()):
+        if self.query_minidump_stackwalk():
             env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
 
         return_code = self.run_command(
             cmd,
             output_timeout=self.test_timeout,
             output_parser=self.job_result_parser,
             env=env
         )
--- a/testing/mozharness/scripts/firefox_media_tests_buildbot.py
+++ b/testing/mozharness/scripts/firefox_media_tests_buildbot.py
@@ -10,121 +10,78 @@ Author: Maja Frydrychowicz
 """
 import copy
 import glob
 import os
 import sys
 
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
-from mozharness.base.log import ERROR, DEBUG, INFO
-from mozharness.base.script import PreScriptAction, PostScriptAction
+from mozharness.base.log import DEBUG, ERROR, INFO
+from mozharness.base.script import PostScriptAction
 from mozharness.mozilla.blob_upload import (
     BlobUploadMixin,
     blobupload_config_options
 )
 from mozharness.mozilla.buildbot import (
     TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
 )
 from mozharness.mozilla.testing.firefox_media_tests import (
-    FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS
+    FirefoxMediaTestsBase, TESTFAILED, SUCCESS
 )
 
-buildbot_media_test_options = [
-    [["--suite"],
-     {"action": "store",
-      "dest": "test_suite",
-      "default": "media-tests",
-      "help": "suite name",
-      }],
-]
-
 
 class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
 
     def __init__(self):
         config_options = copy.deepcopy(blobupload_config_options)
         super(FirefoxMediaTestsBuildbot, self).__init__(
-            config_options=config_options + buildbot_media_test_options,
+            config_options=config_options,
             all_actions=['clobber',
                          'read-buildbot-config',
                          'checkout',
                          'download-and-extract',
                          'create-virtualenv',
                          'install',
                          'run-media-tests',
                          ],
         )
-        c = self.config
-        self.installer_url = c.get('installer_url')
-        self.installer_path = c.get('installer_path')
-        self.binary_path = c.get('binary_path')
-        self.test_packages_url = c.get('test_packages_url')
-
-    @PreScriptAction('create-virtualenv')
-    def _pre_create_virtualenv(self, action):
-        dirs = self.query_abs_dirs()
-        requirements = os.path.join(dirs['abs_test_install_dir'],
-                                    'config',
-                                    'marionette_requirements.txt')
-        if os.access(requirements, os.F_OK):
-            self.register_virtualenv_module(requirements=[requirements],
-                                            two_pass=True)
-        super(FirefoxMediaTestsBuildbot, self)._pre_create_virtualenv(action)
-
-    def query_abs_dirs(self):
-        if self.abs_dirs:
-            return self.abs_dirs
-        dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
-        dirs['abs_blob_upload_dir'] = os.path.join(dirs['abs_work_dir'],
-                                                   'blobber_upload_dir')
-        dirs['abs_test_install_dir'] = os.path.join(dirs['abs_work_dir'],
-                                                    'tests')
-        self.abs_dirs = dirs
-        return self.abs_dirs
-
-    def _query_cmd(self):
-        """ Determine how to call firefox-media-tests """
-        cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
-        # configure logging
-        dirs = self.query_abs_dirs()
-        blob_upload_dir = dirs.get('abs_blob_upload_dir')
-        cmd += ['--gecko-log', os.path.join(blob_upload_dir,
-                                            'gecko.log')]
-        cmd += ['--log-html', os.path.join(blob_upload_dir,
-                                           'media_tests.html')]
-        cmd += ['--log-mach', os.path.join(blob_upload_dir,
-                                           'media_tests_mach.log')]
-
-        test_suite = self.config.get('test_suite')
-        test_manifest = None if test_suite != 'media-youtube-tests' else \
-            os.path.join(dirs['firefox_media_dir'],
-                         'firefox_media_tests',
-                         'playback', 'youtube', 'manifest.ini')
-        config_fmt_args = {
-            'test_manifest': test_manifest,
-        }
-
-        if test_suite not in self.config["suite_definitions"]:
-            self.fatal("%s is not defined in the config!" % test_suite)
-        for s in self.config["suite_definitions"][test_suite]["options"]:
-            cmd.append(s % config_fmt_args)
-
-        return cmd
 
     def run_media_tests(self):
         status = super(FirefoxMediaTestsBuildbot, self).run_media_tests()
         if status == SUCCESS:
             tbpl_status = TBPL_SUCCESS
         else:
             tbpl_status = TBPL_FAILURE
         if status == TESTFAILED:
             tbpl_status = TBPL_WARNING
         self.buildbot_status(tbpl_status)
 
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
+        dirs = {
+            'abs_blob_upload_dir': os.path.join(abs_dirs['abs_work_dir'],
+                                                   'blobber_upload_dir')
+        }
+        abs_dirs.update(dirs)
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def _query_cmd(self):
+        cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
+        dirs = self.query_abs_dirs()
+        # configure logging
+        blob_upload_dir = dirs.get('abs_blob_upload_dir')
+        cmd += ['--gecko-log', os.path.join(blob_upload_dir, 'gecko.log')]
+        cmd += ['--log-html', os.path.join(blob_upload_dir, 'media_tests.html')]
+        cmd += ['--log-mach', os.path.join(blob_upload_dir, 'media_tests_mach.log')]
+        return cmd
+
     @PostScriptAction('run-media-tests')
     def _collect_uploads(self, action, success=None):
         """ Copy extra (log) files to blob upload dir. """
         dirs = self.query_abs_dirs()
         log_dir = dirs.get('abs_log_dir')
         blob_upload_dir = dirs.get('abs_blob_upload_dir')
         if not log_dir or not blob_upload_dir:
             return
new file mode 100755
--- /dev/null
+++ b/testing/mozharness/scripts/firefox_media_tests_jenkins.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** BEGIN LICENSE BLOCK *****
+"""firefox_media_tests_jenkins.py
+
+Author: Syd Polk
+"""
+import copy
+import glob
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.script import PreScriptAction
+from mozharness.mozilla.testing.firefox_media_tests import (
+    FirefoxMediaTestsBase
+)
+
+
+class FirefoxMediaTestsJenkins(FirefoxMediaTestsBase):
+
+    def __init__(self):
+        super(FirefoxMediaTestsJenkins, self).__init__(
+            all_actions=['clobber',
+                         'checkout',
+                         'download-and-extract',
+                         'create-virtualenv',
+                         'install',
+                         'run-media-tests',
+                         ],
+        )
+
+    def _query_cmd(self):
+        cmd = super(FirefoxMediaTestsJenkins, self)._query_cmd()
+
+        dirs = self.query_abs_dirs()
+
+        # configure logging
+        log_dir = dirs.get('abs_log_dir')
+        cmd += ['--gecko-log', os.path.join(log_dir, 'gecko.log')]
+        cmd += ['--log-html', os.path.join(log_dir, 'media_tests.html')]
+        cmd += ['--log-mach', os.path.join(log_dir, 'media_tests_mach.log')]
+
+        return cmd
+
+if __name__ == '__main__':
+    media_test = FirefoxMediaTestsJenkins()
+    media_test.run_and_exit()