Bug 1326047 - Skip unit tests which rely on an instance but Marionette doesn't manage the process. r=ato,maja_zf
authorHenrik Skupin <mail@hskupin.info>
Wed, 04 Jan 2017 13:04:55 +0100
changeset 328261 732e83a0de144e7bd7743180dd369ce177282201
parent 328260 12ef9e56288185646bf4ec5e4c8d0ccbbd4b4ee0
child 328318 891a02f9f66021d341c3ff5435b3539298a6c8dd
push id35738
push userhskupin@mozilla.com
push dateFri, 06 Jan 2017 15:43:33 +0000
treeherderautoland@732e83a0de14 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato, maja_zf
bugs1326047
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1326047 - Skip unit tests which rely on an instance but Marionette doesn't manage the process. r=ato,maja_zf Unit tests which do a restart/quit of the application should not be run if Marionette doesn't handle the instance. Otherwise Marionette is not able to launch the application again for further tests. MozReview-Commit-ID: 8mKylM0r4dM
testing/marionette/driver.js
testing/marionette/harness/marionette_harness/__init__.py
testing/marionette/harness/marionette_harness/marionette_test/__init__.py
testing/marionette/harness/marionette_harness/marionette_test/decorators.py
testing/marionette/harness/marionette_harness/runner/base.py
testing/marionette/harness/marionette_harness/tests/harness_unit/conftest.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
testing/marionette/harness/marionette_harness/tests/unit/test_capabilities.py
testing/marionette/harness/marionette_harness/tests/unit/test_marionette.py
testing/marionette/harness/marionette_harness/tests/unit/test_timeouts.py
testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini
--- a/testing/marionette/driver.js
+++ b/testing/marionette/driver.js
@@ -2666,17 +2666,17 @@ GeckoDriver.prototype.acceptConnections 
   this._server.acceptConnections = cmd.parameters.value;
 }
 
 /**
  * Quits Firefox with the provided flags and tears down the current
  * session.
  */
 GeckoDriver.prototype.quitApplication = function (cmd, resp) {
-  assert.firefox()
+  assert.firefox("Bug 1298921 - In app initiated quit not yet available beside Firefox")
 
   let flags = Ci.nsIAppStartup.eAttemptQuit;
   for (let k of cmd.parameters.flags || []) {
     flags |= Ci.nsIAppStartup[k];
   }
 
   this._server.acceptConnections = false;
   resp.send();
--- a/testing/marionette/harness/marionette_harness/__init__.py
+++ b/testing/marionette/harness/marionette_harness/__init__.py
@@ -4,16 +4,17 @@
 
 __version__ = '4.0.0'
 
 from .marionette_test import (
     CommonTestCase,
     expectedFailure,
     MarionetteTestCase,
     run_if_e10s,
+    run_if_manage_instance,
     skip,
     skip_if_chrome,
     skip_if_desktop,
     skip_if_e10s,
     skip_if_mobile,
     SkipTest,
     skip_unless_protocol,
 )
--- a/testing/marionette/harness/marionette_harness/marionette_test/__init__.py
+++ b/testing/marionette/harness/marionette_harness/marionette_test/__init__.py
@@ -9,16 +9,17 @@ from unittest.case import (
     expectedFailure,
     skip,
     SkipTest,
 )
 
 from .decorators import (
     parameterized,
     run_if_e10s,
+    run_if_manage_instance,
     skip_if_chrome,
     skip_if_desktop,
     skip_if_e10s,
     skip_if_mobile,
     skip_unless_browser_pref,
     skip_unless_protocol,
     with_parameters,
 )
--- a/testing/marionette/harness/marionette_harness/marionette_test/decorators.py
+++ b/testing/marionette/harness/marionette_harness/marionette_test/decorators.py
@@ -64,16 +64,31 @@ def run_if_e10s(reason):
                 """)
                 if multi_process_browser:
                     raise SkipTest(reason)
             return test_item(self, *args, **kwargs)
         return skip_wrapper
     return decorator
 
 
+def run_if_manage_instance(reason):
+    """Decorator which runs a test if Marionette manages the application instance."""
+    def decorator(test_item):
+        if not isinstance(test_item, types.FunctionType):
+            raise Exception('Decorator only supported for functions')
+
+        @functools.wraps(test_item)
+        def skip_wrapper(self, *args, **kwargs):
+            if self.marionette.instance is None:
+                raise SkipTest(reason)
+            return test_item(self, *args, **kwargs)
+        return skip_wrapper
+    return decorator
+
+
 def skip_if_chrome(reason):
     """Decorator which skips a test if chrome context is active."""
     def decorator(test_item):
         if not isinstance(test_item, types.FunctionType):
             raise Exception('Decorator only supported for functions')
 
         @functools.wraps(test_item)
         def skip_wrapper(self, *args, **kwargs):
--- a/testing/marionette/harness/marionette_harness/runner/base.py
+++ b/testing/marionette/harness/marionette_harness/runner/base.py
@@ -941,16 +941,17 @@ class BaseMarionetteTestRunner(object):
 
             filters = []
             if self.test_tags:
                 filters.append(tags(self.test_tags))
 
             values = {
                 "appname": self.appName,
                 "e10s": self.e10s,
+                "manage_instance": self.marionette.instance is not None,
             }
             values.update(mozinfo.info)
 
             manifest_tests = manifest.active_tests(exists=False,
                                                    disabled=True,
                                                    filters=filters,
                                                    **values)
             if len(manifest_tests) == 0:
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/conftest.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/conftest.py
@@ -87,14 +87,14 @@ def mock_httpd(request):
     """ Mock httpd instance """
     httpd = MagicMock(spec=FixtureServer)
     return httpd
 
 
 @pytest.fixture
 def mock_marionette(request):
     """ Mock marionette instance """
-    marionette = MagicMock(spec=Marionette)
+    marionette_class = MagicMock(spec=Marionette)
     if 'has_crashed' in request.funcargnames:
-        marionette.check_for_crash.return_value = request.getfuncargvalue(
+        marionette_class.check_for_crash.return_value = request.getfuncargvalue(
             'has_crashed'
         )
-    return marionette
+    return marionette_class
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -296,16 +296,17 @@ def test_add_test_directory(runner):
         assert test_dir in test['filepath']
     assert len(runner.tests) == 4
 
 
 @pytest.mark.parametrize("test_files_exist", [True, False])
 def test_add_test_manifest(mock_runner, manifest_with_tests, monkeypatch, test_files_exist):
     monkeypatch.setattr('marionette_harness.runner.base.TestManifest',
                         manifest_with_tests.manifest_class)
+    mock_runner.marionette = mock_runner.driverclass()
     with patch('marionette_harness.runner.base.os.path.exists', return_value=test_files_exist):
         if test_files_exist or manifest_with_tests.n_enabled == 0:
             mock_runner.add_test(manifest_with_tests.filepath)
             assert len(mock_runner.tests) == manifest_with_tests.n_enabled
             assert len(mock_runner.manifest_skipped_tests) == manifest_with_tests.n_disabled
             for test in mock_runner.tests:
                 assert test['filepath'].endswith(test['expected'] + '.py')
         else:
@@ -317,16 +318,17 @@ def test_add_test_manifest(mock_runner, 
 def get_kwargs_passed_to_manifest(mock_runner, manifest, monkeypatch, **kwargs):
     '''Helper function for test_manifest_* tests.
     Returns the kwargs passed to the call to manifest.active_tests.'''
     monkeypatch.setattr('marionette_harness.runner.base.TestManifest', manifest.manifest_class)
     monkeypatch.setattr('marionette_harness.runner.base.mozinfo.info',
                         {'mozinfo_key': 'mozinfo_val'})
     for attr in kwargs:
         setattr(mock_runner, attr, kwargs[attr])
+    mock_runner.marionette = mock_runner.driverclass()
     with patch('marionette_harness.runner.base.os.path.exists', return_value=True):
         mock_runner.add_test(manifest.filepath)
     call_args, call_kwargs = manifest.manifest_class().active_tests.call_args
     return call_kwargs
 
 
 def test_manifest_basic_args(mock_runner, manifest, monkeypatch):
     kwargs = get_kwargs_passed_to_manifest(mock_runner, manifest, monkeypatch)
--- a/testing/marionette/harness/marionette_harness/tests/unit/test_capabilities.py
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_capabilities.py
@@ -38,20 +38,21 @@ class TestCapabilities(MarionetteTestCas
         self.assertIn("acceptInsecureCerts", self.caps)
         self.assertFalse(self.caps["acceptInsecureCerts"])
 
     def test_additional_capabilities(self):
         self.assertIn("moz:processID", self.caps)
         self.assertEqual(self.caps["moz:processID"], self.appinfo["processID"])
         self.assertEqual(self.marionette.process_id, self.appinfo["processID"])
 
-        current_profile = self.marionette.instance.runner.profile.profile
         self.assertIn("moz:profile", self.caps)
-        self.assertEqual(self.caps["moz:profile"], current_profile)
-        self.assertEqual(self.marionette.profile, current_profile)
+        if self.marionette.instance is not None:
+            current_profile = self.marionette.instance.runner.profile.profile
+            self.assertEqual(self.caps["moz:profile"], current_profile)
+            self.assertEqual(self.marionette.profile, current_profile)
 
         self.assertIn("moz:accessibilityChecks", self.caps)
         self.assertFalse(self.caps["moz:accessibilityChecks"])
 
     def test_we_can_pass_in_capabilities_on_session_start(self):
         self.marionette.delete_session()
         capabilities = {"desiredCapabilities": {"somethingAwesome": "cake"}}
         self.marionette.start_session(capabilities)
--- a/testing/marionette/harness/marionette_harness/tests/unit/test_marionette.py
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_marionette.py
@@ -2,31 +2,33 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import itertools
 import time
 
 from marionette_driver import errors
 
-from marionette_harness import MarionetteTestCase
+from marionette_harness import MarionetteTestCase, run_if_manage_instance, skip_if_mobile
 
 
 class TestMarionette(MarionetteTestCase):
 
     def test_correct_test_name(self):
         """Test that the correct test name gets set."""
         expected_test_name = '{module}.py {cls}.{func}'.format(
             module=__name__,
             cls=self.__class__.__name__,
             func=self.test_correct_test_name.__name__,
         )
 
         self.assertEqual(self.marionette.test_name, expected_test_name)
 
+    @run_if_manage_instance("Only runnable if Marionette manages the instance")
+    @skip_if_mobile("Bug 1322993 - Missing temporary folder")
     def test_wait_for_port_non_existing_process(self):
         """Test that wait_for_port doesn't run into a timeout if instance is not running."""
         self.marionette.quit()
         self.assertIsNotNone(self.marionette.instance.runner.returncode)
         start_time = time.time()
         self.assertFalse(self.marionette.wait_for_port(timeout=5))
         self.assertLess(time.time() - start_time, 5)
 
--- a/testing/marionette/harness/marionette_harness/tests/unit/test_timeouts.py
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_timeouts.py
@@ -5,17 +5,17 @@
 from marionette_driver.by import By
 from marionette_driver.errors import (
     MarionetteException,
     NoSuchElementException,
     ScriptTimeoutException,
 )
 from marionette_driver.marionette import HTMLElement
 
-from marionette_harness import MarionetteTestCase
+from marionette_harness import MarionetteTestCase, run_if_manage_instance, skip_if_mobile
 
 
 class TestTimeouts(MarionetteTestCase):
     def tearDown(self):
         self.marionette.timeout.reset()
         MarionetteTestCase.tearDown(self)
 
     def test_page_timeout_notdefinetimeout_pass(self):
@@ -50,16 +50,18 @@ class TestTimeouts(MarionetteTestCase):
 
     def test_search_timeout_found(self):
         test_html = self.marionette.absolute_url("test.html")
         self.marionette.navigate(test_html)
         button = self.marionette.find_element(By.ID, "createDivButton")
         button.click()
         self.assertRaises(NoSuchElementException, self.marionette.find_element, By.ID, "newDiv")
 
+    @run_if_manage_instance("Only runnable if Marionette manages the instance")
+    @skip_if_mobile("Bug 1322993 - Missing temporary folder")
     def test_reset_timeout(self):
         timeouts = [getattr(self.marionette.timeout, f) for f in (
             'implicit', 'page_load', 'script',)]
 
         def do_check(callback):
             for timeout in timeouts:
                 timeout = 10000
                 self.assertEqual(timeout, 10000)
--- a/testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini
+++ b/testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini
@@ -90,24 +90,25 @@ skip-if = appname == 'fennec'
 skip-if = true # Bug 925688
 [test_chrome_async_finish.js]
 [test_screen_orientation.py]
 [test_errors.py]
 
 [test_execute_isolate.py]
 [test_click_scrolling.py]
 [test_profile_management.py]
+skip-if = manage_instance == false || appname == 'fennec' # Bug 1298921 and bug 1322993
 [test_quit_restart.py]
-skip-if = appname == 'fennec' # Bug 1298921
+skip-if = manage_instance == false || appname == 'fennec' # Bug 1298921 and bug 1322993
 [test_set_window_size.py]
 skip-if = os == "linux" || appname == 'fennec' # Bug 1085717
 [test_with_using_context.py]
 
 [test_modal_dialogs.py]
-skip-if = appname == 'fennec'
+skip-if = manage_instance == false || appname == 'fennec' # Bug 1328696
 [test_key_actions.py]
 [test_mouse_action.py]
 skip-if = appname == 'fennec'
 [test_teardown_context_preserved.py]
 [test_file_upload.py]
 skip-if = appname == 'fennec' || os == "win" # http://bugs.python.org/issue14574
 
 [test_execute_sandboxes.py]
@@ -118,9 +119,10 @@ skip-if = appname == 'fennec' || os == "
 
 [test_chrome.py]
 skip-if = appname == 'fennec'
 
 [test_addons.py]
 
 [test_select.py]
 [test_crash.py]
+skip-if = manage_instance == false || appname == 'fennec' # Bug 1298921 and bug 1322993
 [test_localization.py]