Bug 1640747 - Add test-level support for handling exceptions while running the tests r=acreskey
authorTarek Ziadé <tarek@mozilla.com>
Tue, 26 May 2020 13:35:00 +0000
changeset 532213 23629b6a120b001b0f51634f3465b3a528b245e5
parent 532212 dabdcdd6264befbf5b1be937b847e288349a1a5d
child 532214 7291cf51007bb64ac83883848b11acd08bde72b2
push id117069
push usertziade@mozilla.com
push dateTue, 26 May 2020 18:07:27 +0000
treeherderautoland@23629b6a120b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersacreskey
bugs1640747
milestone78.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1640747 - Add test-level support for handling exceptions while running the tests r=acreskey This patch adds the `on_exception` hook, that will let developers drive the runner behavior when a layer fails. Differential Revision: https://phabricator.services.mozilla.com/D76794
python/mozperftest/mozperftest/environment.py
python/mozperftest/mozperftest/tests/data/hook.py
python/mozperftest/mozperftest/tests/data/hook_raises.py
python/mozperftest/mozperftest/tests/data/hook_resume.py
python/mozperftest/mozperftest/tests/test_browsertime.py
python/mozperftest/mozperftest/tests/test_environment.py
--- a/python/mozperftest/mozperftest/environment.py
+++ b/python/mozperftest/mozperftest/environment.py
@@ -80,18 +80,30 @@ class MachEnvironment:
         # freeze args  (XXX do we need to freeze more?)
         self._saved_mach_args = copy.deepcopy(self._mach_args)
 
     def unfreeze(self):
         self._mach_args = self._saved_mach_args
         self._saved_mach_args = None
 
     def run(self, metadata):
+        has_exc_handler = self.has_hook("on_exception")
         for layer in self.layers:
-            metadata = layer(metadata)
+            try:
+                metadata = layer(metadata)
+            except Exception as e:
+                if has_exc_handler:
+                    # if the hook returns True, we abort and return
+                    # without error. If it returns False, we continue
+                    # the loop. The hook can also raise an exception or
+                    # re-raise this exception.
+                    if not self.run_hook("on_exception", layer, e):
+                        return metadata
+                else:
+                    raise
         return metadata
 
     def __enter__(self):
         for layer in self.layers:
             layer.__enter__()
         return self
 
     def __exit__(self, type, value, traceback):
@@ -119,14 +131,17 @@ class MachEnvironment:
         if hooks.exists():
             spec = importlib.util.spec_from_file_location("hooks", str(hooks))
             hooks = importlib.util.module_from_spec(spec)
             spec.loader.exec_module(hooks)
             self._hooks = hooks
         else:
             raise IOError(str(hooks))
 
-    def run_hook(self, name, **kw):
+    def has_hook(self, name):
+        return hasattr(self._hooks, name)
+
+    def run_hook(self, name, *args, **kw):
         if self._hooks is None:
             return
         if not hasattr(self._hooks, name):
             return
-        return getattr(self._hooks, name)(self, **kw)
+        return getattr(self._hooks, name)(self, *args, **kw)
--- a/python/mozperftest/mozperftest/tests/data/hook.py
+++ b/python/mozperftest/mozperftest/tests/data/hook.py
@@ -1,2 +1,7 @@
 def doit(env):
     return "OK"
+
+
+def on_exception(env, layer, exc):
+    # swallow the error and abort the run
+    return False
new file mode 100644
--- /dev/null
+++ b/python/mozperftest/mozperftest/tests/data/hook_raises.py
@@ -0,0 +1,3 @@
+def on_exception(env, layer, exc):
+    # re-raise
+    raise exc
new file mode 100644
--- /dev/null
+++ b/python/mozperftest/mozperftest/tests/data/hook_resume.py
@@ -0,0 +1,3 @@
+def on_exception(env, layer, exc):
+    # swallow the error and resume
+    return True
--- a/python/mozperftest/mozperftest/tests/test_browsertime.py
+++ b/python/mozperftest/mozperftest/tests/test_browsertime.py
@@ -60,16 +60,17 @@ def test_browser(*mocked):
     assert "--one 1" in cmd
     assert "--two 2" in cmd
 
     results = metadata.get_results()
     assert len(results) == 1
     assert set(list(results[0].keys())) - set(["name", "results"]) == set()
     assert results[0]["name"] == "Example"
 
+
 @mock.patch("mozperftest.browser.browsertime.runner.install_package")
 @mock.patch(
     "mozperftest.browser.noderunner.NodeRunner.verify_node_install", new=lambda x: True
 )
 @mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
 @mock.patch(
     "mozperftest.browser.browsertime.runner.BrowsertimeRunner._setup_node_packages",
     new=lambda x, y: None,
--- a/python/mozperftest/mozperftest/tests/test_environment.py
+++ b/python/mozperftest/mozperftest/tests/test_environment.py
@@ -43,10 +43,58 @@ def test_layers():
 
 def test_context():
     mach, metadata, env = get_running_env()
     env.layers = [mock.MagicMock(), mock.MagicMock()]
     with env:
         env.run(metadata)
 
 
+class FailureException(Exception):
+    pass
+
+
+class Failure:
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args, **kw):
+        pass
+
+    def __call__(self, *args, **kw):
+        raise FailureException()
+
+
+def test_exception_return():
+    # the last layer is not called, the error is swallowed
+    hooks = str(Path(HERE, "data", "hook.py"))
+    mach, metadata, env = get_running_env(hooks=hooks)
+    last_layer = mock.MagicMock()
+    env.layers = [mock.MagicMock(), Failure(), last_layer]
+    with env:
+        env.run(metadata)
+    last_layer.assert_not_called()
+
+
+def test_exception_resume():
+    # the last layer is called, the error is swallowed
+    hooks = str(Path(HERE, "data", "hook_resume.py"))
+    mach, metadata, env = get_running_env(hooks=hooks)
+    last_layer = mock.MagicMock()
+    env.layers = [mock.MagicMock(), Failure(), last_layer]
+    with env:
+        env.run(metadata)
+    last_layer.assert_called()
+
+
+def test_exception_raised():
+    # the error is raised
+    hooks = str(Path(HERE, "data", "hook_raises.py"))
+    mach, metadata, env = get_running_env(hooks=hooks)
+    last_layer = mock.MagicMock()
+    env.layers = [mock.MagicMock(), Failure(), last_layer]
+    with env, pytest.raises(FailureException):
+        env.run(metadata)
+    last_layer.assert_not_called()
+
+
 if __name__ == "__main__":
     mozunit.main()