Bug 1230279 - Develop new mozharness script for media-tests for Jenkins; r?maja_zf, whimboo draft
authorSyd Polk <spolk@mozilla.com>
Fri, 08 Jan 2016 14:46:29 -0600
changeset 320192 d5e28ba3c853020dc406610904297bb5894fefa2
parent 316994 ad16863d1d45bd3fd7906c76fa1ac1e12d24a133
child 512703 21f75ea59c518d19d9560cbe34e6ec48068ad45f
push id9151
push userspolk@mozilla.com
push dateFri, 08 Jan 2016 22:42:49 +0000
reviewersmaja_zf, whimboo
bugs1230279
milestone46.0a1
Bug 1230279 - Develop new mozharness script for media-tests for Jenkins; r?maja_zf, whimboo
testing/mozharness/configs/mediatests/buildbot_posix_config.py
testing/mozharness/configs/mediatests/buildbot_windows_config.py
testing/mozharness/configs/mediatests/jenkins_config.py
testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
testing/mozharness/scripts/firefox_media_tests_buildbot.py
testing/mozharness/scripts/firefox_media_tests_jenkins.py
--- a/testing/mozharness/configs/mediatests/buildbot_posix_config.py
+++ b/testing/mozharness/configs/mediatests/buildbot_posix_config.py
@@ -36,20 +36,17 @@ config = {
          "https://blobupload.elasticbeanstalk.com",
     ],
     "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
     "download_minidump_stackwalk": True,
     "download_symbols": "ondemand",
 
     "firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
     "firefox_media_branch": 'master',
-    "firefox_media_rev": '49b500b30b80372a6c678ec7d0a2b074844f5e84',
-    "firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
-    "firefox_ui_branch": 'mozilla-central',
-    "firefox_ui_rev": '32be49d74e1d10c6bf087235b1d6753c1b840bc4',
+    "firefox_media_rev": '0830e972e4b95fef3507207fc6bce028da27f2d3',
 
     "suite_definitions": {
         "media-tests": {
             "options": [],
         },
         "media-youtube-tests": {
             "options": [
                 "%(test_manifest)s"
--- a/testing/mozharness/configs/mediatests/buildbot_windows_config.py
+++ b/testing/mozharness/configs/mediatests/buildbot_windows_config.py
@@ -14,20 +14,16 @@ config = {
         'python': 'c:/mozilla-build/python27/python',
         'virtualenv': ['c:/mozilla-build/python27/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
         'hg': 'c:/mozilla-build/hg/hg',
         'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(),
                        '%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()],
         'tooltool.py': [sys.executable, 'C:/mozilla-build/tooltool.py'],
         'gittool.py': [sys.executable,
                        os.path.join(external_tools_path, 'gittool.py')],
-        'hgtool.py': [sys.executable,
-                      os.path.join(external_tools_path, 'hgtool.py')],
-
-
     },
 
     "find_links": [
         "http://pypi.pvt.build.mozilla.org/pub",
         "http://pypi.pub.build.mozilla.org/pub",
     ],
     "pip_index": False,
 
@@ -47,20 +43,17 @@ config = {
     ],
     "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
     "in_tree_config": "config/mozharness/marionette.py",
     "download_minidump_stackwalk": True,
     "download_symbols": "ondemand",
 
     "firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
     "firefox_media_branch": 'master',
-    "firefox_media_rev": '49b500b30b80372a6c678ec7d0a2b074844f5e84',
-    "firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
-    "firefox_ui_branch": 'mozilla-central',
-    "firefox_ui_rev": '32be49d74e1d10c6bf087235b1d6753c1b840bc4',
+    "firefox_media_rev": '0830e972e4b95fef3507207fc6bce028da27f2d3',
 
     "suite_definitions": {
         "media-tests": {
             "options": [],
         },
         "media-youtube-tests": {
             "options": [
                 "%(test_manifest)s"
new file mode 100755
--- /dev/null
+++ b/testing/mozharness/configs/mediatests/jenkins_config.py
@@ -0,0 +1,72 @@
+# Default configuration as used by Mozmill CI (Jenkins)
+
+import os
+import platform
+import sys
+
+import mozharness
+
+
+external_tools_path = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+    'external_tools',
+)
+
+config = {
+    'env': {
+        'PIP_TRUSTED_HOST': 'pypi.pub.build.mozilla.org',
+    },
+
+    # PIP
+    'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
+    'pip_index': False,
+
+    # mozcrash support
+    'download_minidump_stackwalk': True,
+    'download_symbols': 'ondemand',
+    'download_tooltool': True,
+
+    # Version control information
+    'firefox_media_repo': 'https://github.com/mjzffr/firefox-media-tests.git',
+    'firefox_media_branch': 'master',
+
+    # Default test suite
+    'test_suite': 'media-tests',
+
+    'suite_definitions': {
+        'media-tests': {
+            'options': [],
+        },
+        'media-youtube-tests': {
+            'options': [
+                '%(test_manifest)s'
+            ],
+        },
+    },
+
+    'default_actions': [
+        'clobber',
+        'checkout',
+        'download-and-extract',
+        'create-virtualenv',
+        'install',
+        'run-media-tests',
+    ],
+
+}
+
+# General local variable overwrite
+# Bug 1227079 - Python executable eeded to get it executed on Windows
+if platform.system() == 'windows':
+    gittool = [
+        sys.executable,
+        os.path.join(external_tools_path, 'gittool.py')
+    ]
+else:
+    gittool = os.path.join(external_tools_path, 'gittool.py')
+
+exes = {
+    'gittool.py' : gittool,
+}
+
+config['exes'] = exes
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
@@ -1,21 +1,20 @@
 #!/usr/bin/env python
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** BEGIN LICENSE BLOCK *****
-"""firefox_media_tests.py
 
-Author: Maja Frydrychowicz
-"""
 import copy
+import glob
 import os
 import re
+import urlparse
 
 from mozharness.base.log import ERROR, WARNING
 from mozharness.base.script import PreScriptAction
 from mozharness.mozilla.testing.testbase import (TestingMixin,
                                                  testing_config_options)
 from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
 from mozharness.mozilla.vcstools import VCSToolsScript
 
@@ -67,33 +66,24 @@ media_test_config_options = [
         'dest': 'firefox_media_branch',
         'default': 'master',
         'help': 'which branch to use for firefox_media_tests',
     }],
     [['--firefox-media-rev'], {
         'dest': 'firefox_media_rev',
         'help': 'which firefox_media_tests revision to use',
     }],
-    [['--firefox-ui-repo'], {
-        'dest': 'firefox_ui_repo',
-        'default': 'https://github.com/mozilla/firefox-ui-tests.git',
-        'help': 'which firefox_ui_tests repo to use',
-    }],
-    [['--firefox-ui-branch'], {
-        'dest': 'firefox_ui_branch',
-        'default': 'master',
-        'help': 'which branch to use for firefox_ui_tests',
-    }],
-    [['--firefox-ui-rev'], {
-        'dest': 'firefox_ui_rev',
-        'help': 'which firefox_ui_tests revision to use',
-    }],
+    [["--suite"],
+     {"action": "store",
+      "dest": "test_suite",
+      "default": "media-tests",
+      "help": "suite name",
+      }],
 ] + (copy.deepcopy(testing_config_options))
 
-
 class JobResultParser(TestSummaryOutputParserHelper):
     """ Parses test output to determine overall result."""
     def __init__(self, **kwargs):
         super(JobResultParser, self).__init__(**kwargs)
         self.return_code = 0
         # External-resource errors that should not count as test failures
         self.exception_re = re.compile(r'^TEST-UNEXPECTED-ERROR.*'
                                        r'TimeoutException: Error loading page,'
@@ -133,80 +123,96 @@ class FirefoxMediaTestsBase(TestingMixin
      ]
 
     def __init__(self, config_options=None, all_actions=None,
                  default_actions=None, **kwargs):
         self.config_options = media_test_config_options + (config_options or [])
         actions = [
             'clobber',
             'checkout',
+            'download-and-extract',
             'create-virtualenv',
+            'install',
             'run-media-tests',
         ]
         super(FirefoxMediaTestsBase, self).__init__(
             config_options=self.config_options,
             all_actions=all_actions or actions,
             default_actions=default_actions or actions,
             **kwargs
         )
         c = self.config
+
         self.media_urls = c.get('media_urls')
         self.profile = c.get('profile')
         self.test_timeout = int(c.get('test_timeout'))
         self.tests = c.get('tests')
         self.e10s = c.get('e10s')
+        self.installer_url = c.get('installer_url')
+        self.installer_path = c.get('installer_path')
+        self.binary_path = c.get('binary_path')
+        self.test_packages_url = c.get('test_packages_url')
+        self.test_url = c.get('test_url')
 
     @PreScriptAction('create-virtualenv')
     def _pre_create_virtualenv(self, action):
         dirs = self.query_abs_dirs()
-        requirements_file = os.path.join(dirs['firefox_media_dir'],
+        marionette_requirements = os.path.join(dirs['abs_test_install_dir'],
+                                    'config',
+                                    'marionette_requirements.txt')
+        if os.access(marionette_requirements, os.F_OK):
+            self.register_virtualenv_module(requirements=[marionette_requirements],
+                                            two_pass=True)
+
+        media_tests_requirements = os.path.join(dirs['firefox_media_dir'],
                                          'requirements.txt')
-        if os.path.isfile(requirements_file):
-            self.register_virtualenv_module(requirements=[requirements_file])
-        self.register_virtualenv_module(name='firefox-ui-tests',
-                                        url=dirs['firefox_ui_dir'])
-        self.register_virtualenv_module(name='firefox-media-tests',
-                                        url=dirs['firefox_media_dir'])
+
+        if os.access(media_tests_requirements, os.F_OK):
+            self.register_virtualenv_module(requirements=[media_tests_requirements],
+                                            two_pass=True)
+
+    def download_and_extract(self):
+        """Overriding method from TestingMixin until firefox-media-tests are in tree.
+
+        Right now we only care about the installer and symbolds.
+
+        """
+        self._download_installer()
+
+        if self.config.get('download_symbols'):
+            self._download_and_extract_symbols()
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
         abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs()
         dirs = {
             'firefox_media_dir': os.path.join(abs_dirs['abs_work_dir'],
                                               'firefox-media-tests')
         }
-        dirs['firefox_ui_dir'] = os.path.join(dirs['firefox_media_dir'],
-                                              'firefox-ui-tests')
+        dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'],
+                                                    'tests')
         abs_dirs.update(dirs)
         self.abs_dirs = abs_dirs
         return self.abs_dirs
 
     @PreScriptAction('checkout')
     def _pre_checkout(self, action):
         super(FirefoxMediaTestsBase, self)._pre_checkout(action)
         c = self.config
         dirs = self.query_abs_dirs()
         self.firefox_media_vc = {
             'branch': c['firefox_media_branch'],
             'repo': c['firefox_media_repo'],
-            'revision': c['firefox_media_rev'],
             'dest': dirs['firefox_media_dir'],
-        }
-        self.firefox_ui_vc = {
-            'branch': c['firefox_ui_branch'],
-            'repo': c['firefox_ui_repo'],
-            'revision': c['firefox_ui_rev'],
-            'dest': dirs['firefox_ui_dir']
+            'revision' : c.get('firefox_media_rev', 'default')
         }
 
     def checkout(self):
-        revision = self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
-        if revision:
-            self.vcs_checkout(vcs='gittool', **self.firefox_ui_vc)
+        self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
 
     def _query_cmd(self):
         """ Determine how to call firefox-media-tests """
         if not self.binary_path:
             self.fatal("Binary path could not be determined. "
                        "Should be set by default during 'install' action.")
         dirs = self.query_abs_dirs()
         venv_python_path = self.query_python_path()
@@ -221,29 +227,68 @@ class FirefoxMediaTestsBase(TestingMixin
             cmd += ['--urls', self.media_urls]
         if self.profile:
             cmd += ['--profile', self.profile]
         if self.tests:
             cmd.append(self.tests)
         if self.e10s:
             cmd.append('--e10s')
 
+        test_suite = self.config.get('test_suite')
+        if test_suite not in self.config["suite_definitions"]:
+            self.fatal("%s is not defined in the config!" % test_suite)
+
+        test_manifest = None if test_suite != 'media-youtube-tests' else \
+            os.path.join(dirs['firefox_media_dir'],
+                         'firefox_media_tests',
+                         'playback', 'youtube', 'manifest.ini')
+        config_fmt_args = {
+            'test_manifest': test_manifest,
+        }
+
+        for s in self.config["suite_definitions"][test_suite]["options"]:
+            cmd.append(s % config_fmt_args)
+
         return cmd
 
+    def query_minidump_stackwalk(self):
+        """We don't have an extracted test package available to get the manifest file.
+
+        So we have to explicitely download the latest version of the manifest from the
+        mozilla-central repository and feed it into the query_minidump_stackwalk() method.
+
+        We can remove this whole method once our tests are part of the tree.
+
+        """
+        manifest_path = None
+
+        if os.environ.get('MINIDUMP_STACKWALK') or self.config.get('download_minidump_stackwalk'):
+            tooltool_manifest = self.query_minidump_tooltool_manifest()
+            url_base = 'https://hg.mozilla.org/mozilla-central/raw-file/default/testing/'
+
+            dirs = self.query_abs_dirs()
+            manifest_path = os.path.join(dirs['abs_work_dir'], 'releng.manifest')
+            try:
+                self.download_file(urlparse.urljoin(url_base, tooltool_manifest),
+                                   manifest_path)
+            except Exception as e:
+                self.fatal('Download of tooltool manifest file failed: %s' % e.message)
+
+        return super(FirefoxMediaTestsBase, self).query_minidump_stackwalk(manifest=manifest_path)
+
     def run_media_tests(self):
         cmd = self._query_cmd()
         self.job_result_parser = JobResultParser(
             config=self.config,
             log_obj=self.log_obj,
             error_list=self.error_list
         )
 
         env = self.query_env()
-        if (not os.environ.get('MINIDUMP_STACKWALK') and
-                self.query_minidump_stackwalk()):
+        if self.query_minidump_stackwalk():
             env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
 
         return_code = self.run_command(
             cmd,
             output_timeout=self.test_timeout,
             output_parser=self.job_result_parser,
             env=env
         )
--- a/testing/mozharness/scripts/firefox_media_tests_buildbot.py
+++ b/testing/mozharness/scripts/firefox_media_tests_buildbot.py
@@ -10,121 +10,78 @@ Author: Maja Frydrychowicz
 """
 import copy
 import glob
 import os
 import sys
 
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
-from mozharness.base.log import ERROR, DEBUG, INFO
-from mozharness.base.script import PreScriptAction, PostScriptAction
+from mozharness.base.log import DEBUG, ERROR, INFO
+from mozharness.base.script import PostScriptAction
 from mozharness.mozilla.blob_upload import (
     BlobUploadMixin,
     blobupload_config_options
 )
 from mozharness.mozilla.buildbot import (
     TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
 )
 from mozharness.mozilla.testing.firefox_media_tests import (
-    FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS
+    FirefoxMediaTestsBase, TESTFAILED, SUCCESS
 )
 
-buildbot_media_test_options = [
-    [["--suite"],
-     {"action": "store",
-      "dest": "test_suite",
-      "default": "media-tests",
-      "help": "suite name",
-      }],
-]
-
 
 class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
 
     def __init__(self):
         config_options = copy.deepcopy(blobupload_config_options)
         super(FirefoxMediaTestsBuildbot, self).__init__(
-            config_options=config_options + buildbot_media_test_options,
+            config_options=config_options,
             all_actions=['clobber',
                          'read-buildbot-config',
                          'checkout',
                          'download-and-extract',
                          'create-virtualenv',
                          'install',
                          'run-media-tests',
                          ],
         )
-        c = self.config
-        self.installer_url = c.get('installer_url')
-        self.installer_path = c.get('installer_path')
-        self.binary_path = c.get('binary_path')
-        self.test_packages_url = c.get('test_packages_url')
-
-    @PreScriptAction('create-virtualenv')
-    def _pre_create_virtualenv(self, action):
-        dirs = self.query_abs_dirs()
-        requirements = os.path.join(dirs['abs_test_install_dir'],
-                                    'config',
-                                    'marionette_requirements.txt')
-        if os.access(requirements, os.F_OK):
-            self.register_virtualenv_module(requirements=[requirements],
-                                            two_pass=True)
-        super(FirefoxMediaTestsBuildbot, self)._pre_create_virtualenv(action)
-
-    def query_abs_dirs(self):
-        if self.abs_dirs:
-            return self.abs_dirs
-        dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
-        dirs['abs_blob_upload_dir'] = os.path.join(dirs['abs_work_dir'],
-                                                   'blobber_upload_dir')
-        dirs['abs_test_install_dir'] = os.path.join(dirs['abs_work_dir'],
-                                                    'tests')
-        self.abs_dirs = dirs
-        return self.abs_dirs
-
-    def _query_cmd(self):
-        """ Determine how to call firefox-media-tests """
-        cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
-        # configure logging
-        dirs = self.query_abs_dirs()
-        blob_upload_dir = dirs.get('abs_blob_upload_dir')
-        cmd += ['--gecko-log', os.path.join(blob_upload_dir,
-                                            'gecko.log')]
-        cmd += ['--log-html', os.path.join(blob_upload_dir,
-                                           'media_tests.html')]
-        cmd += ['--log-mach', os.path.join(blob_upload_dir,
-                                           'media_tests_mach.log')]
-
-        test_suite = self.config.get('test_suite')
-        test_manifest = None if test_suite != 'media-youtube-tests' else \
-            os.path.join(dirs['firefox_media_dir'],
-                         'firefox_media_tests',
-                         'playback', 'youtube', 'manifest.ini')
-        config_fmt_args = {
-            'test_manifest': test_manifest,
-        }
-
-        if test_suite not in self.config["suite_definitions"]:
-            self.fatal("%s is not defined in the config!" % test_suite)
-        for s in self.config["suite_definitions"][test_suite]["options"]:
-            cmd.append(s % config_fmt_args)
-
-        return cmd
 
     def run_media_tests(self):
         status = super(FirefoxMediaTestsBuildbot, self).run_media_tests()
         if status == SUCCESS:
             tbpl_status = TBPL_SUCCESS
         else:
             tbpl_status = TBPL_FAILURE
         if status == TESTFAILED:
             tbpl_status = TBPL_WARNING
         self.buildbot_status(tbpl_status)
 
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
+        dirs = {
+            'abs_blob_upload_dir': os.path.join(abs_dirs['abs_work_dir'],
+                                                   'blobber_upload_dir')
+        }
+        abs_dirs.update(dirs)
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def _query_cmd(self):
+        cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
+        dirs = self.query_abs_dirs()
+        # configure logging
+        blob_upload_dir = dirs.get('abs_blob_upload_dir')
+        cmd += ['--gecko-log', os.path.join(blob_upload_dir, 'gecko.log')]
+        cmd += ['--log-html', os.path.join(blob_upload_dir, 'media_tests.html')]
+        cmd += ['--log-mach', os.path.join(blob_upload_dir, 'media_tests_mach.log')]
+        return cmd
+
     @PostScriptAction('run-media-tests')
     def _collect_uploads(self, action, success=None):
         """ Copy extra (log) files to blob upload dir. """
         dirs = self.query_abs_dirs()
         log_dir = dirs.get('abs_log_dir')
         blob_upload_dir = dirs.get('abs_blob_upload_dir')
         if not log_dir or not blob_upload_dir:
             return
new file mode 100755
--- /dev/null
+++ b/testing/mozharness/scripts/firefox_media_tests_jenkins.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** BEGIN LICENSE BLOCK *****
+"""firefox_media_tests_jenkins.py
+
+Author: Syd Polk
+"""
+import copy
+import glob
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.script import PreScriptAction
+from mozharness.mozilla.testing.firefox_media_tests import (
+    FirefoxMediaTestsBase
+)
+
+
+class FirefoxMediaTestsJenkins(FirefoxMediaTestsBase):
+
+    def __init__(self):
+        super(FirefoxMediaTestsJenkins, self).__init__(
+            all_actions=['clobber',
+                         'checkout',
+                         'download-and-extract',
+                         'create-virtualenv',
+                         'install',
+                         'run-media-tests',
+                         ],
+        )
+
+    @PreScriptAction('create-virtualenv')
+    def _pre_create_virtualenv(self, action):
+        dirs = self.query_abs_dirs()
+        requirements = os.path.join(dirs['abs_test_install_dir'],
+                                    'config',
+                                    'marionette_requirements.txt')
+        if os.access(requirements, os.F_OK):
+            self.register_virtualenv_module(requirements=[requirements],
+                                            two_pass=True)
+        super(FirefoxMediaTestsJenkins, self)._pre_create_virtualenv(action)
+
+if __name__ == '__main__':
+    media_test = FirefoxMediaTestsJenkins()
+    media_test.run_and_exit()