Bug 1212609 - [mozharness] Update firefox-ui-tests script to use common.tests.zip file. r=jlund a=testonly
authorHenrik Skupin <mail@hskupin.info>
Thu, 21 Jan 2016 21:34:52 +0100
changeset 298438 b9cf5e85cf7f683d288bd1e0370af8a6be13190a
parent 298437 41159a21b1aaee33952d73720715f45aeb6b04e3
child 298439 468638b88f3cec55a7062412ff8d978b3de0dd2d
push id8951
push userhskupin@mozilla.com
push dateThu, 21 Jan 2016 20:42:49 +0000
treeherdermozilla-aurora@ec2bd9b72622 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlund, testonly
bugs1212609
milestone45.0a2
Bug 1212609 - [mozharness] Update firefox-ui-tests script to use common.tests.zip file. r=jlund a=testonly
testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
testing/mozharness/scripts/firefox_ui_tests/functional.py
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
@@ -94,85 +94,92 @@ firefox_ui_update_harness_config_options
 ]
 
 firefox_ui_update_config_options = firefox_ui_update_harness_config_options \
     + copy.deepcopy(firefox_ui_tests_config_options)
 
 
 class FirefoxUITests(TestingMixin, VCSToolsScript):
 
-    cli_script = 'runtests.py'
+    # Needs to be overwritten in sub classes
+    cli_script = None
 
     def __init__(self, config_options=None,
                  all_actions=None, default_actions=None,
                  *args, **kwargs):
         config_options = config_options or firefox_ui_tests_config_options
         actions = [
             'clobber',
             'download-and-extract',
-            'checkout',  # keep until firefox-ui-tests are located in tree
             'create-virtualenv',
             'install',
             'run-tests',
             'uninstall',
         ]
 
         super(FirefoxUITests, self).__init__(
             config_options=config_options,
             all_actions=all_actions or actions,
             default_actions=default_actions or actions,
             *args, **kwargs)
 
+        # As long as we don't run on buildbot the following properties have be set on our own
+        self.installer_url = self.config.get('installer_url')
+        self.installer_path = self.config.get('installer_path')
+        self.test_packages_url = self.config.get('test_packages_url')
+        self.test_url = self.config.get('test_url')
+
         self.reports = {'html': 'report.html', 'xunit': 'report.xml'}
 
         self.firefox_ui_repo = self.config['firefox_ui_repo']
         self.firefox_ui_branch = self.config.get('firefox_ui_branch')
 
-        if not self.firefox_ui_branch:
+        if not self.test_url and not self.test_packages_url and not self.firefox_ui_branch:
             self.fatal(
-                'Please specify --firefox-ui-branch. Valid values can be found '
-                'in here https://github.com/mozilla/firefox-ui-tests/branches')
-
-        # As long as we don't run on buildbot the installers are not handled by TestingMixin
-        self.installer_url = self.config.get('installer_url')
-        self.installer_path = self.config.get('installer_path')
+                'You must use --test-url, --test-packages-url, or --firefox-ui-branch (valid '
+                'values can be found at: https://github.com/mozilla/firefox-ui-tests/branches)')
 
     @PreScriptAction('create-virtualenv')
     def _pre_create_virtualenv(self, action):
         dirs = self.query_abs_dirs()
 
-        # List of exact versions of mozbase packages which are known to work
-        requirements_file = os.path.join(dirs['fx_ui_dir'], 'requirements.txt')
-        if os.path.isfile(requirements_file):
-            self.register_virtualenv_module(requirements=[requirements_file])
+        # If tests are used from common.tests.zip install every Python package
+        # via the single requirements file
+        if self.test_packages_url or self.test_url:
+            test_install_dir = dirs.get('abs_test_install_dir',
+                                        os.path.join(dirs['abs_work_dir'], 'tests'))
+            requirements_file = os.path.join(test_install_dir,
+                                             'config', 'firefox_ui_requirements.txt')
+            if os.path.isfile(requirements_file):
+                self.register_virtualenv_module(requirements=[requirements_file])
 
-        # Optional packages to be installed, e.g. for Jenkins
-        if self.config.get('virtualenv_modules'):
-            for module in self.config['virtualenv_modules']:
-                self.register_virtualenv_module(module)
-
-        self.register_virtualenv_module('firefox-ui-tests', url=dirs['fx_ui_dir'])
+        # We have a non-packaged version of Firefox UI tests. So install requirements
+        # and the firefox-ui-tests package separately
+        else:
+            # Register all modules for firefox-ui-tests including all dependencies
+            # as strict versions to ensure newer releases won't break something
+            requirements_file = os.path.join(dirs.get('abs_test_install_dir',
+                                                      os.path.join(dirs['abs_work_dir'], 'tests')),
+                                             'requirements.txt')
+            if os.path.isfile(requirements_file):
+                self.register_virtualenv_module(requirements=[requirements_file])
 
-    @PreScriptAction('checkout')
-    def _pre_checkout(self, action):
-        if not self.firefox_ui_branch:
-            self.fatal(
-                'Please specify --firefox-ui-branch. Valid values can be found '
-                'in here https://github.com/mozilla/firefox-ui-tests/branches')
+            # Optional packages to be installed, e.g. for Jenkins
+            if self.config.get('virtualenv_modules'):
+                for module in self.config['virtualenv_modules']:
+                    self.register_virtualenv_module(module)
 
     def checkout(self):
-        """
-        We checkout firefox_ui_tests and update to the right branch
-        for it.
-        """
+        """Clone the firefox-ui-tests repository."""
         dirs = self.query_abs_dirs()
 
         self.vcs_checkout(
             repo=self.firefox_ui_repo,
-            dest=dirs['fx_ui_dir'],
+            dest=dirs.get('abs_test_install_dir',
+                          os.path.join(dirs['abs_work_dir'], 'tests')),
             branch=self.firefox_ui_branch,
             vcs='gittool',
             env=self.query_env(),
         )
 
     def clobber(self):
         """Delete the working directory"""
         super(FirefoxUITests, self).clobber()
@@ -188,34 +195,40 @@ class FirefoxUITests(TestingMixin, VCSTo
         for report in self.reports:
             self.copy_to_upload_dir(os.path.join(dirs['abs_reports_dir'], self.reports[report]),
                                     dest=os.path.join('reports', self.reports[report]),
                                     short_desc='%s log' % self.reports[report],
                                     long_desc='%s log' % self.reports[report],
                                     max_backups=self.config.get("log_max_rotate", 0))
 
     def download_and_extract(self):
-        """Overriding method from TestingMixin until firefox-ui-tests are in tree.
+        """Overriding method from TestingMixin until firefox-ui-tests are in tree and
+        supported across all branches.
 
-        Right now we only care about the installer and symbolds.
+        We use the test_packages_url command line argument to check where to get the
+        harness, puppeteer, and tests from and how to set them up.
 
         """
-        self._download_installer()
+        if self.test_packages_url or self.test_url:
+            super(FirefoxUITests, self).download_and_extract()
 
-        if self.config.get('download_symbols'):
-            self._download_and_extract_symbols()
+        else:
+            self.checkout()
+            self._download_installer()
+
+            if self.config.get('download_symbols'):
+                self._download_and_extract_symbols()
 
     def query_abs_dirs(self):
         if self.abs_dirs:
             return self.abs_dirs
 
         abs_dirs = VCSToolsScript.query_abs_dirs(self)
         abs_dirs.update({
             'abs_reports_dir': os.path.join(abs_dirs['base_work_dir'], 'reports'),
-            'fx_ui_dir': os.path.join(abs_dirs['abs_work_dir'], 'firefox_ui_tests'),
         })
         self.abs_dirs = abs_dirs
 
         return self.abs_dirs
 
     def query_harness_args(self, extra_harness_config_options=None):
         """Collects specific update test related command line arguments.
 
@@ -233,24 +246,26 @@ class FirefoxUITests(TestingMixin, VCSTo
                 if type(name) is bool:
                     args.append(option[0][0])
                 else:
                     args.extend([option[0][0], self.config[dest]])
 
         return args
 
     def query_minidump_stackwalk(self):
-        """We don't have an extracted test package available to get the manifest file.
+        """Download the minidump stackwalk binary.
 
-        So we have to explicitely download the latest version of the manifest from the
-        mozilla-central repository and feed it into the query_minidump_stackwalk() method.
-
-        We can remove this whole method once our tests are part of the tree.
+        We can remove this whole method once we no longer need the github repository.
 
         """
+        # If the test package is available use it
+        if self.test_packages_url or self.test_url:
+            return super(FirefoxUITests, self).query_minidump_stackwalk()
+
+        # Otherwise grab the manifest file from hg.mozilla.org
         manifest_path = None
 
         if self.config.get('download_minidump_stackwalk'):
             tooltool_manifest = self.query_minidump_tooltool_manifest()
             url_base = 'https://hg.mozilla.org/mozilla-central/raw-file/default/testing/'
 
             dirs = self.query_abs_dirs()
             manifest_path = os.path.join(dirs['abs_work_dir'], 'releng.manifest')
@@ -266,19 +281,23 @@ class FirefoxUITests(TestingMixin, VCSTo
     def _upload_reports_post_run(self):
         if self.config.get("copy_reports_post_run", True):
             self.copy_reports_to_upload_dir()
 
     def run_test(self, binary_path, env=None, marionette_port=2828):
         """All required steps for running the tests against an installer."""
         dirs = self.query_abs_dirs()
 
+        # Import the harness to retrieve the location of the cli scripts
+        import firefox_ui_harness
+
         cmd = [
             self.query_python_path(),
-            os.path.join(dirs['fx_ui_dir'], 'firefox_ui_harness', self.cli_script),
+            os.path.join(os.path.dirname(firefox_ui_harness.__file__),
+                         self.cli_script),
             '--binary', binary_path,
             '--address', 'localhost:{}'.format(marionette_port),
 
             # Use the work dir to get temporary data stored
             '--workspace', dirs['abs_work_dir'],
 
             # logging options
             '--gecko-log=-',  # output from the gecko process redirected to stdout
@@ -325,16 +344,21 @@ class FirefoxUITests(TestingMixin, VCSTo
     def run_tests(self):
         """Run all the tests"""
         return self.run_test(
             binary_path=self.binary_path,
             env=self.query_env(),
         )
 
 
+class FirefoxUIFunctionalTests(FirefoxUITests):
+
+    cli_script = 'cli_functional.py'
+
+
 class FirefoxUIUpdateTests(FirefoxUITests):
 
     cli_script = 'cli_update.py'
 
     def __init__(self, config_options=None, *args, **kwargs):
         config_options = config_options or firefox_ui_update_config_options
 
         FirefoxUITests.__init__(self, config_options=config_options,
--- a/testing/mozharness/scripts/firefox_ui_tests/functional.py
+++ b/testing/mozharness/scripts/firefox_ui_tests/functional.py
@@ -7,14 +7,14 @@
 
 
 import os
 import sys
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
 
-from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUITests
+from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUIFunctionalTests
 
 
 if __name__ == '__main__':
-    myScript = FirefoxUITests()
+    myScript = FirefoxUIFunctionalTests()
     myScript.run_and_exit()