Bug 1276974 - Add tests for BaseMarionetteTestRunner.add_test; r=maja_zf
authorAnjana Vakil <anjanavakil@gmail.com>
Wed, 08 Jun 2016 20:11:16 +0200
changeset 302238 f852be67d6252bbf21b0a08258e9dea129b26410
parent 302237 fcd860b7546f72dfbc95dff5f0fc443b244485b1
child 302239 5d1285d0c22151079e8e3424a9c6429b3642471d
push id78658
push usermjzffr@gmail.com
push dateTue, 21 Jun 2016 21:05:52 +0000
treeherdermozilla-inbound@f852be67d625 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmaja_zf
bugs1276974
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1276974 - Add tests for BaseMarionetteTestRunner.add_test; r=maja_zf Test that paths to test modules, directories, and manifest files are processed correctly by BaseMarionetteTestRunner.add_test. Individual test modules should be added even if they have invalid test names. If a directory path is provided, all validly-named test modules discovered by walking the dir should be added. Manifest files should be parsed and tests added/skipped accordingly. MozReview-Commit-ID: 2GJFEGGcEW2
testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
--- a/testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
@@ -289,11 +289,75 @@ def test_increment_crash_count_in_run_te
 
 @pytest.mark.parametrize("has_crashed", [True, False])
 def test_record_crash(runner, has_crashed, mock_marionette):
     with patch.object(runner, 'marionette', mock_marionette):
         assert runner.record_crash() == has_crashed
         _check_crash_counts(has_crashed, runner, runner.marionette)
 
 
+def test_add_test_module(runner):
+    tests = ['test_something.py', 'testSomething.js', 'bad_test.py']
+    assert len(runner.tests) == 0
+    for test in tests:
+        with patch ('os.path.abspath') as abspath:
+            abspath.return_value = test
+            runner.add_test(test)
+        assert abspath.called
+        assert {'filepath': test,
+                'expected': 'pass',
+                'test_container': None} in runner.tests
+    # add_test doesn't validate module names; 'bad_test.py' gets through
+    assert len(runner.tests) == 3
+
+def test_add_test_directory(runner):
+    test_dir = 'path/to/tests'
+    dir_contents = [
+        (test_dir, ('subdir',), ('test_a.py', 'test_a.js', 'bad_test_a.py', 'bad_test_a.js')),
+        (test_dir + '/subdir', (), ('test_b.py', 'test_b.js', 'bad_test_a.py', 'bad_test_b.js')),
+    ]
+    tests = list(dir_contents[0][2] + dir_contents[1][2])
+    assert len(runner.tests) == 0
+    with patch('os.path.isdir') as isdir:
+        # Need to use side effect to make isdir return True for test_dir and False for tests
+        isdir.side_effect = [True] + [False for i in tests]
+        with patch('os.walk') as walk:
+            walk.return_value = dir_contents
+            runner.add_test(test_dir)
+    assert isdir.called and walk.called
+    for test in runner.tests:
+        assert test_dir in test['filepath']
+    assert len(runner.tests) == 4
+
+def test_add_test_manifest(runner):
+    runner._device, runner._appName = 'fake_device', 'fake_app'
+    manifest = "/path/to/fake/manifest.ini"
+    active_tests = [{'expected': 'pass',
+                     'path': u'/path/to/fake/test_expected_pass.py'},
+                    {'expected': 'fail',
+                     'path': u'/path/to/fake/test_expected_fail.py'},
+                    {'disabled': 'skip-if: true # "testing disabled test"',
+                     'expected': 'pass',
+                     'path': u'/path/to/fake/test_disabled.py'}]
+    with patch.multiple('marionette.runner.base.TestManifest',
+                        read=DEFAULT, active_tests=DEFAULT) as mocks:
+            mocks['active_tests'].return_value = active_tests
+            with pytest.raises(IOError) as err:
+                runner.add_test(manifest)
+            assert "does not exist" in err.value.message
+            assert mocks['read'].call_count == mocks['active_tests'].call_count == 1
+            runner.tests, runner.manifest_skipped_tests = [], []
+            with patch('marionette.runner.base.os.path.exists', return_value=True):
+                runner.add_test(manifest)
+            assert mocks['read'].call_count == mocks['active_tests'].call_count == 2
+    assert len(runner.tests) == 2
+    assert len(runner.manifest_skipped_tests) == 1
+    for test in runner.tests:
+        assert test['filepath'].endswith(('test_expected_pass.py', 'test_expected_fail.py'))
+        if test['filepath'].endswith('test_expected_fail.py'):
+            assert test['expected'] == 'fail'
+        else:
+            assert test['expected'] == 'pass'
+
+
 if __name__ == '__main__':
     import sys
     sys.exit(pytest.main(['--verbose', __file__]))