Bug 1275269 - Improve & test BaseMarionetteTestRunner._add_tests; r=maja_zf
authorAnjana Vakil <anjanavakil@gmail.com>
Thu, 23 Jun 2016 17:01:20 +0200
changeset 302755 0cbfbfe2a193c56e656b38cd054a09ea5f4d3076
parent 302754 b60487f638fd937ec630c546df55aebc12aa6795
child 302756 f0d6274311d480ea61630fa5998511e66f82a774
push id30376
push usercbook@mozilla.com
push dateTue, 28 Jun 2016 14:09:36 +0000
treeherdermozilla-central@e45890951ce7 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmaja_zf
bugs1275269
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1275269 - Improve & test BaseMarionetteTestRunner._add_tests; r=maja_zf Rewrite the filename verification in BaseMarionetteTestRunner._add_tests to check for ".py"/".js" extensions as well as "test_" prefix and to allow for JavaScript filenames like 'testSomething.js'. Use regular expression matching instead of str.startswith and such. In test_marionette_runner.py, test _add_tests (called by run_tests): - Test that _add_tests populates self.tests with correct tests - Test that file names are accepted/rejected appropriately MozReview-Commit-ID: 7SzYGJcHNus
testing/marionette/harness/marionette/runner/base.py
testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
--- a/testing/marionette/harness/marionette/runner/base.py
+++ b/testing/marionette/harness/marionette/runner/base.py
@@ -4,16 +4,17 @@
 
 from argparse import ArgumentParser
 
 import json
 import mozinfo
 import moznetwork
 import os
 import random
+import re
 import socket
 import sys
 import time
 import traceback
 import unittest
 import warnings
 import mozprofile
 
@@ -776,26 +777,28 @@ setReq.onerror = function() {
                 self.logger.info("starting httpd")
                 self.start_httpd(need_external_ip)
                 self.marionette.baseurl = self.httpd.get_url()
                 self.logger.info("running httpd on %s" % self.marionette.baseurl)
             else:
                 self.marionette.baseurl = self.server_root
                 self.logger.info("using remote content from %s" % self.marionette.baseurl)
 
-
     def _add_tests(self, tests):
         for test in tests:
             self.add_test(test)
 
-        invalid_tests = \
-            [t['filepath'] for t in self.tests
-             if not os.path.basename(t['filepath']).startswith('test_')]
+        pattern = re.compile("^test(((_.+?)+?\.((py)|(js)))|(([A-Z].*?)+?\.js))$")
+        def is_valid(test):
+            filename = os.path.basename(test['filepath'])
+            return pattern.match(filename)
+        invalid_tests = [t['filepath'] for t in self.tests if not is_valid(t)]
         if invalid_tests:
-            raise Exception("Tests file names must start with 'test_'."
+            raise Exception("Test file names must be of the form "
+                            "'test_something.py', 'test_something.js', or 'testSomething.js'."
                             " Invalid test names:\n  %s"
                             % '\n  '.join(invalid_tests))
 
     def _log_skipped_tests(self):
         for test in self.manifest_skipped_tests:
             name = os.path.basename(test['path'])
             self.logger.test_start(name)
             self.logger.test_end(name,
--- a/testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
@@ -395,11 +395,36 @@ def test_initialize_test_run(mock_runner
     assert "len(tests)" in str(test_exc.traceback[-1].statement)
     with pytest.raises(AssertionError) as hndl_exc:
         mock_runner.test_handlers = []
         mock_runner.run_tests(tests)
     assert "test_handlers" in str(hndl_exc.traceback[-1].statement)
     assert mock_runner.reset_test_stats.call_count == 1
 
 
+def test_add_tests(mock_runner):
+    assert len(mock_runner.tests) == 0
+    fake_tests = ["test_" + i + ".py" for i in "abc"]
+    with patch('marionette.runner.base.mozversion.get_version'):
+        mock_runner.run_tests(fake_tests)
+    assert len(mock_runner.tests) == 3
+    for (test_name, added_test) in zip(fake_tests, mock_runner.tests):
+        assert added_test['filepath'].endswith(test_name)
+
+
+def test_catch_invalid_test_names(runner):
+    good_tests = [u'test_ok.py', u'test_is_ok.py', u'test_is_ok.js', u'testIsOk.js']
+    bad_tests = [u'bad_test.py', u'testbad.py', u'_test_bad.py', u'testBad.notjs',
+                 u'test_bad.notpy', u'test_bad', u'testbad.js', u'badtest.js',
+                 u'test.py', u'test_.py', u'test.js', u'test_.js']
+    with pytest.raises(Exception) as exc:
+        runner._add_tests(good_tests + bad_tests)
+    msg = exc.value.message
+    assert "Test file names must be of the form" in msg
+    for bad_name in bad_tests:
+        assert bad_name in msg
+    for good_name in good_tests:
+        assert good_name not in msg
+
+
 if __name__ == '__main__':
     import sys
     sys.exit(pytest.main(['--verbose', __file__]))