Bug 1326047 - Skip unit tests which rely on an instance but Marionette doesn't manage the process. r=ato, r=maja_zf, a=test-only
authorHenrik Skupin <mail@hskupin.info>
Wed, 04 Jan 2017 13:04:55 +0100
changeset 353409 4e8559ebed8f6230b413404125d31077e168df6a
parent 353408 1c83035673d00cad18406588f739645ab58c116b
child 353410 0252fcd07201b644e4fce16aa01e11dfb599d065
push id6795
push userjlund@mozilla.com
push dateMon, 23 Jan 2017 14:19:46 +0000
treeherdermozilla-esr52@76101b503191 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato, maja_zf, test-only
bugs1326047
milestone52.0a2
Bug 1326047 - Skip unit tests which rely on an instance but Marionette doesn't manage the process. r=ato, r=maja_zf, a=test-only Unit tests which do a restart/quit of the application should not be run if Marionette doesn't handle the instance. Otherwise Marionette is not able to launch the application again for further tests. MozReview-Commit-ID: 8mKylM0r4dM
testing/marionette/driver.js
testing/marionette/harness/marionette_harness/__init__.py
testing/marionette/harness/marionette_harness/marionette_test/__init__.py
testing/marionette/harness/marionette_harness/marionette_test/decorators.py
testing/marionette/harness/marionette_harness/runner/base.py
testing/marionette/harness/marionette_harness/tests/harness_unit/conftest.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
testing/marionette/harness/marionette_harness/tests/unit/test_capabilities.py
testing/marionette/harness/marionette_harness/tests/unit/test_marionette.py
testing/marionette/harness/marionette_harness/tests/unit/test_timeouts.py
testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini
--- a/testing/marionette/driver.js
+++ b/testing/marionette/driver.js
@@ -2658,17 +2658,17 @@ GeckoDriver.prototype.acceptConnections 
   this._server.acceptConnections = cmd.parameters.value;
 }
 
 /**
  * Quits Firefox with the provided flags and tears down the current
  * session.
  */
 GeckoDriver.prototype.quitApplication = function (cmd, resp) {
-  assert.firefox()
+  assert.firefox("Bug 1298921 - In app initiated quit not yet available beside Firefox")
 
   let flags = Ci.nsIAppStartup.eAttemptQuit;
   for (let k of cmd.parameters.flags || []) {
     flags |= Ci.nsIAppStartup[k];
   }
 
   this._server.acceptConnections = false;
   resp.send();
--- a/testing/marionette/harness/marionette_harness/__init__.py
+++ b/testing/marionette/harness/marionette_harness/__init__.py
@@ -4,16 +4,17 @@
 
 __version__ = '4.0.0'
 
 from .marionette_test import (
     CommonTestCase,
     expectedFailure,
     MarionetteTestCase,
     run_if_e10s,
+    run_if_manage_instance,
     skip,
     skip_if_chrome,
     skip_if_desktop,
     skip_if_e10s,
     skip_if_mobile,
     SkipTest,
     skip_unless_protocol,
 )
--- a/testing/marionette/harness/marionette_harness/marionette_test/__init__.py
+++ b/testing/marionette/harness/marionette_harness/marionette_test/__init__.py
@@ -9,16 +9,17 @@ from unittest.case import (
     expectedFailure,
     skip,
     SkipTest,
 )
 
 from .decorators import (
     parameterized,
     run_if_e10s,
+    run_if_manage_instance,
     skip_if_chrome,
     skip_if_desktop,
     skip_if_e10s,
     skip_if_mobile,
     skip_unless_browser_pref,
     skip_unless_protocol,
     with_parameters,
 )
--- a/testing/marionette/harness/marionette_harness/marionette_test/decorators.py
+++ b/testing/marionette/harness/marionette_harness/marionette_test/decorators.py
@@ -64,16 +64,31 @@ def run_if_e10s(reason):
                 """)
                 if multi_process_browser:
                     raise SkipTest(reason)
             return test_item(self, *args, **kwargs)
         return skip_wrapper
     return decorator
 
 
+def run_if_manage_instance(reason):
+    """Decorator which runs a test if Marionette manages the application instance."""
+    def decorator(test_item):
+        if not isinstance(test_item, types.FunctionType):
+            raise Exception('Decorator only supported for functions')
+
+        @functools.wraps(test_item)
+        def skip_wrapper(self, *args, **kwargs):
+            if self.marionette.instance is None:
+                raise SkipTest(reason)
+            return test_item(self, *args, **kwargs)
+        return skip_wrapper
+    return decorator
+
+
 def skip_if_chrome(reason):
     """Decorator which skips a test if chrome context is active."""
     def decorator(test_item):
         if not isinstance(test_item, types.FunctionType):
             raise Exception('Decorator only supported for functions')
 
         @functools.wraps(test_item)
         def skip_wrapper(self, *args, **kwargs):
--- a/testing/marionette/harness/marionette_harness/runner/base.py
+++ b/testing/marionette/harness/marionette_harness/runner/base.py
@@ -939,16 +939,17 @@ class BaseMarionetteTestRunner(object):
 
             filters = []
             if self.test_tags:
                 filters.append(tags(self.test_tags))
 
             values = {
                 "appname": self.appName,
                 "e10s": self.e10s,
+                "manage_instance": self.marionette.instance is not None,
             }
             values.update(mozinfo.info)
 
             manifest_tests = manifest.active_tests(exists=False,
                                                    disabled=True,
                                                    filters=filters,
                                                    **values)
             if len(manifest_tests) == 0:
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/conftest.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/conftest.py
@@ -91,14 +91,14 @@ def mock_httpd(request):
     """ Mock httpd instance """
     httpd = MagicMock(spec=FixtureServer)
     return httpd
 
 
 @pytest.fixture
 def mock_marionette(request):
     """ Mock marionette instance """
-    marionette = MagicMock(spec=Marionette)
+    marionette_class = MagicMock(spec=Marionette)
     if 'has_crashed' in request.funcargnames:
-        marionette.check_for_crash.return_value = request.getfuncargvalue(
+        marionette_class.check_for_crash.return_value = request.getfuncargvalue(
             'has_crashed'
         )
-    return marionette
+    return marionette_class
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -291,16 +291,17 @@ def test_add_test_directory(runner):
         assert test_dir in test['filepath']
     assert len(runner.tests) == 4
 
 
 @pytest.mark.parametrize("test_files_exist", [True, False])
 def test_add_test_manifest(mock_runner, manifest_with_tests, monkeypatch, test_files_exist):
     monkeypatch.setattr('marionette_harness.runner.base.TestManifest',
                         manifest_with_tests.manifest_class)
+    mock_runner.marionette = mock_runner.driverclass()
     with patch('marionette_harness.runner.base.os.path.exists', return_value=test_files_exist):
         if test_files_exist or manifest_with_tests.n_enabled == 0:
             mock_runner.add_test(manifest_with_tests.filepath)
             assert len(mock_runner.tests) == manifest_with_tests.n_enabled
             assert len(mock_runner.manifest_skipped_tests) == manifest_with_tests.n_disabled
             for test in mock_runner.tests:
                 assert test['filepath'].endswith(test['expected'] + '.py')
         else:
@@ -312,16 +313,17 @@ def test_add_test_manifest(mock_runner, 
 def get_kwargs_passed_to_manifest(mock_runner, manifest, monkeypatch, **kwargs):
     '''Helper function for test_manifest_* tests.
     Returns the kwargs passed to the call to manifest.active_tests.'''
     monkeypatch.setattr('marionette_harness.runner.base.TestManifest', manifest.manifest_class)
     monkeypatch.setattr('marionette_harness.runner.base.mozinfo.info',
                         {'mozinfo_key': 'mozinfo_val'})
     for attr in kwargs:
         setattr(mock_runner, attr, kwargs[attr])
+    mock_runner.marionette = mock_runner.driverclass()
     with patch('marionette_harness.runner.base.os.path.exists', return_value=True):
         mock_runner.add_test(manifest.filepath)
     call_args, call_kwargs = manifest.manifest_class().active_tests.call_args
     return call_kwargs
 
 
 def test_manifest_basic_args(mock_runner, manifest, monkeypatch):
     kwargs = get_kwargs_passed_to_manifest(mock_runner, manifest, monkeypatch)
--- a/testing/marionette/harness/marionette_harness/tests/unit/test_capabilities.py
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_capabilities.py
@@ -38,20 +38,21 @@ class TestCapabilities(MarionetteTestCas
         self.assertIn("acceptInsecureCerts", self.caps)
         self.assertFalse(self.caps["acceptInsecureCerts"])
 
     def test_additional_capabilities(self):
         self.assertIn("moz:processID", self.caps)
         self.assertEqual(self.caps["moz:processID"], self.appinfo["processID"])
         self.assertEqual(self.marionette.process_id, self.appinfo["processID"])
 
-        current_profile = self.marionette.instance.runner.profile.profile
         self.assertIn("moz:profile", self.caps)
-        self.assertEqual(self.caps["moz:profile"], current_profile)
-        self.assertEqual(self.marionette.profile, current_profile)
+        if self.marionette.instance is not None:
+            current_profile = self.marionette.instance.runner.profile.profile
+            self.assertEqual(self.caps["moz:profile"], current_profile)
+            self.assertEqual(self.marionette.profile, current_profile)
 
         self.assertIn("moz:accessibilityChecks", self.caps)
         self.assertFalse(self.caps["moz:accessibilityChecks"])
 
     def test_we_can_pass_in_capabilities_on_session_start(self):
         self.marionette.delete_session()
         capabilities = {"desiredCapabilities": {"somethingAwesome": "cake"}}
         self.marionette.start_session(capabilities)
--- a/testing/marionette/harness/marionette_harness/tests/unit/test_marionette.py
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_marionette.py
@@ -2,31 +2,33 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import itertools
 import time
 
 from marionette_driver import errors
 
-from marionette_harness import MarionetteTestCase
+from marionette_harness import MarionetteTestCase, run_if_manage_instance, skip_if_mobile
 
 
 class TestMarionette(MarionetteTestCase):
 
     def test_correct_test_name(self):
         """Test that the correct test name gets set."""
         expected_test_name = '{module}.py {cls}.{func}'.format(
             module=__name__,
             cls=self.__class__.__name__,
             func=self.test_correct_test_name.__name__,
         )
 
         self.assertEqual(self.marionette.test_name, expected_test_name)
 
+    @run_if_manage_instance("Only runnable if Marionette manages the instance")
+    @skip_if_mobile("Bug 1322993 - Missing temporary folder")
     def test_wait_for_port_non_existing_process(self):
         """Test that wait_for_port doesn't run into a timeout if instance is not running."""
         self.marionette.quit()
         self.assertIsNotNone(self.marionette.instance.runner.returncode)
         start_time = time.time()
         self.assertFalse(self.marionette.wait_for_port(timeout=5))
         self.assertLess(time.time() - start_time, 5)
 
--- a/testing/marionette/harness/marionette_harness/tests/unit/test_timeouts.py
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_timeouts.py
@@ -5,17 +5,17 @@
 from marionette_driver.by import By
 from marionette_driver.errors import (
     MarionetteException,
     NoSuchElementException,
     ScriptTimeoutException,
 )
 from marionette_driver.marionette import HTMLElement
 
-from marionette_harness import MarionetteTestCase
+from marionette_harness import MarionetteTestCase, run_if_manage_instance, skip_if_mobile
 
 
 class TestTimeouts(MarionetteTestCase):
     def tearDown(self):
         self.marionette.timeout.reset()
         MarionetteTestCase.tearDown(self)
 
     def test_page_timeout_notdefinetimeout_pass(self):
@@ -50,16 +50,18 @@ class TestTimeouts(MarionetteTestCase):
 
     def test_search_timeout_found(self):
         test_html = self.marionette.absolute_url("test.html")
         self.marionette.navigate(test_html)
         button = self.marionette.find_element(By.ID, "createDivButton")
         button.click()
         self.assertRaises(NoSuchElementException, self.marionette.find_element, By.ID, "newDiv")
 
+    @run_if_manage_instance("Only runnable if Marionette manages the instance")
+    @skip_if_mobile("Bug 1322993 - Missing temporary folder")
     def test_reset_timeout(self):
         timeouts = [getattr(self.marionette.timeout, f) for f in (
             'implicit', 'page_load', 'script',)]
 
         def do_check(callback):
             for timeout in timeouts:
                 timeout = 10000
                 self.assertEqual(timeout, 10000)
--- a/testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini
+++ b/testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini
@@ -90,24 +90,25 @@ skip-if = appname == 'fennec'
 skip-if = true # Bug 925688
 [test_chrome_async_finish.js]
 [test_screen_orientation.py]
 [test_errors.py]
 
 [test_execute_isolate.py]
 [test_click_scrolling.py]
 [test_profile_management.py]
+skip-if = manage_instance == false || appname == 'fennec' # Bug 1298921 and bug 1322993
 [test_quit_restart.py]
-skip-if = appname == 'fennec' # Bug 1298921
+skip-if = manage_instance == false || appname == 'fennec' # Bug 1298921 and bug 1322993
 [test_set_window_size.py]
 skip-if = os == "linux" || appname == 'fennec' # Bug 1085717
 [test_with_using_context.py]
 
 [test_modal_dialogs.py]
-skip-if = appname == 'fennec'
+skip-if = manage_instance == false || appname == 'fennec' # Bug 1328696
 [test_key_actions.py]
 [test_mouse_action.py]
 skip-if = appname == 'fennec'
 [test_teardown_context_preserved.py]
 [test_file_upload.py]
 skip-if = appname == 'fennec' || os == "win" # http://bugs.python.org/issue14574
 
 [test_execute_sandboxes.py]
@@ -118,9 +119,10 @@ skip-if = appname == 'fennec' || os == "
 
 [test_chrome.py]
 skip-if = appname == 'fennec'
 
 [test_addons.py]
 
 [test_select.py]
 [test_crash.py]
+skip-if = manage_instance == false || appname == 'fennec' # Bug 1298921 and bug 1322993
 [test_localization.py]