Bug 1339178 - Use pytest to run python-tests, r=davehunt
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 29 Aug 2017 14:50:33 -0400
changeset 378007 3f58f4dc8303cb08e66907b3d403e8d2a1cf3234
parent 378006 3cfed550091b330f6f15f9e34198712ba3a23ee8
child 378008 787441c9619d67540aaeb5bdafb0ddc88e654b7f
push id50145
push userahalberstadt@mozilla.com
push dateThu, 31 Aug 2017 16:06:45 +0000
treeherderautoland@3f58f4dc8303 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1339178
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1339178 - Use pytest to run python-tests, r=davehunt This switches most tests over to use pytest as the runner instead of unittest (taking advantage of the fact that pytest can run unittest based tests). There were a couple tests that had failures when swithing to pytest: config/tests/unit-expandlibs.py xpcom/idl-parser/xpidl/runtests.py For these tests, I added a runwith='unittest' argument so that they still run the same way as before. Once we fix them to use pytest, the unittest logic in mozunit.py can be deleted. MozReview-Commit-ID: Gcsz6z8MeOi
config/mozunit.py
config/tests/unit-expandlibs.py
python/mach_commands.py
python/mozbuild/mozbuild/test/test_pythonutil.py
python/mozlint/test/test_cli.py
python/mozlint/test/test_formatters.py
python/mozlint/test/test_parser.py
python/mozlint/test/test_roller.py
python/mozlint/test/test_types.py
python/mozlint/test/test_vcs.py
testing/marionette/harness/marionette_harness/tests/harness_unit/pytest.ini
testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
testing/mochitest/tests/python/test_basic_mochitest_plain.py
testing/mochitest/tests/python/test_get_active_tests.py
testing/mozbase/mozcrash/tests/test.py
testing/mozbase/mozlog/tests/test_logger.py
xpcom/idl-parser/xpidl/runtests.py
--- a/config/mozunit.py
+++ b/config/mozunit.py
@@ -4,16 +4,18 @@
 
 import inspect
 import os
 import sys
 import unittest
 from StringIO import StringIO
 from unittest import TextTestRunner as _TestRunner, TestResult as _TestResult
 
+import pytest
+
 '''Helper to make python unit tests report the way that the Mozilla
 unit test infrastructure expects tests to report.
 
 Usage:
 
 import mozunit
 
 if __name__ == '__main__':
@@ -204,9 +206,24 @@ class MockedOpen(object):
         abspath = normcase(os.path.abspath(p) + os.sep)
         if any(f.startswith(abspath) for f in self.files):
             return True
 
         return self._orig_path_exists(p)
 
 
 def main(*args, **kwargs):
-    unittest.main(testRunner=MozTestRunner(), *args, **kwargs)
+    runwith = kwargs.pop('runwith', 'pytest')
+
+    if runwith == 'unittest':
+        unittest.main(testRunner=MozTestRunner(), *args, **kwargs)
+    else:
+        args = list(args)
+        if os.environ.get('MACH_STDOUT_ISATTY') and not any(a.startswith('--color') for a in args):
+            args.append('--color=yes')
+
+        module = __import__('__main__')
+        args.extend([
+            '--verbose',
+            '-p', 'mozlog.pytest_mozlog.plugin',
+            module.__file__,
+        ])
+        sys.exit(pytest.main(args))
--- a/config/tests/unit-expandlibs.py
+++ b/config/tests/unit-expandlibs.py
@@ -423,9 +423,9 @@ class TestSymbolOrder(unittest.TestCase)
         config.LD_PRINT_ICF_SECTIONS = '-Wl,--print-icf-sections'
         args = ExpandArgsMore(['foo', '-bar', 'bar.o', 'foo.o'])
         self.assertEqual(args._getOrderedSections(['hello', '_Z6barbazv']), ['.text.hello', '.text.hi', '.text.hot._Z6barbazv'])
         self.assertEqual(args._getOrderedSections(['_ZThn4_6foobarv', 'hi', '_Z6barbazv']), ['.text._Z6foobarv', '.text._ZThn4_6foobarv', '.text.hi', '.text.hello', '.text.hot._Z6barbazv'])
         subprocess.Popen = subprocess_popen
 
 
 if __name__ == '__main__':
-    mozunit.main()
+    mozunit.main(runwith='unittest')
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -201,17 +201,17 @@ class MachCommands(MachCommandBase):
         def _line_handler(line):
             if not file_displayed_test:
                 output = ('Ran' in line or 'collected' in line or
                           line.startswith('TEST-'))
                 if output:
                     file_displayed_test.append(True)
 
             # Hack to make sure treeherder highlights pytest failures
-            if line.endswith('FAILED'):
+            if 'FAILED' in line.rsplit(' ', 1)[-1]:
                 line = line.replace('FAILED', 'TEST-UNEXPECTED-FAIL')
 
             _log(line)
 
         _log(test_path)
         cmd = [self.virtualenv_manager.python_path, test_path]
         env = os.environ.copy()
         env[b'PYTHONDONTWRITEBYTECODE'] = b'1'
--- a/python/mozbuild/mozbuild/test/test_pythonutil.py
+++ b/python/mozbuild/mozbuild/test/test_pythonutil.py
@@ -7,17 +7,18 @@ from mozunit import main
 import os
 import unittest
 
 
 class TestIterModules(unittest.TestCase):
     def test_iter_modules_in_path(self):
         mozbuild_path = os.path.normcase(os.path.dirname(os.path.dirname(__file__)))
         paths = list(iter_modules_in_path(mozbuild_path))
-        self.assertEquals(sorted(paths), [
+        self.assertEquals(set(paths), set([
             os.path.join(os.path.abspath(mozbuild_path), '__init__.py'),
             os.path.join(os.path.abspath(mozbuild_path), 'pythonutil.py'),
+            os.path.join(os.path.abspath(mozbuild_path), 'test', '__init__.py'),
             os.path.join(os.path.abspath(mozbuild_path), 'test', 'test_pythonutil.py'),
-        ])
+        ]))
 
 
 if __name__ == '__main__':
     main()
--- a/python/mozlint/test/test_cli.py
+++ b/python/mozlint/test/test_cli.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import sys
 from distutils.spawn import find_executable
 
+import mozunit
 import pytest
 
 from mozlint import cli
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 
 @pytest.fixture
@@ -53,9 +53,9 @@ def test_cli_run_with_edit(run, parser, 
     assert "foobar.js: line 2, col 1, Error" in out[2]
 
     del os.environ['EDITOR']
     with pytest.raises(SystemExit):
         parser.parse_args(['--edit'])
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_formatters.py
+++ b/python/mozlint/test/test_formatters.py
@@ -1,18 +1,18 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import json
-import sys
 from collections import defaultdict
 
+import mozunit
 import pytest
 
 from mozlint import ResultContainer
 from mozlint import formatters
 
 
 EXPECTED = {
     'compact': {
@@ -99,9 +99,9 @@ def test_json_formatter(results):
 
     slots = ResultContainer.__slots__
     for errors in formatted.values():
         for err in errors:
             assert all(s in err for s in slots)
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_parser.py
+++ b/python/mozlint/test/test_parser.py
@@ -1,15 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import sys
 
+import mozunit
 import pytest
 
 from mozlint.parser import Parser
 from mozlint.errors import (
     LinterNotFound,
     LinterParseError,
 )
 
@@ -53,9 +53,9 @@ def test_parse_invalid_linter(parse, lin
 
 
 def test_parse_non_existent_linter(parse):
     with pytest.raises(LinterNotFound):
         parse('missing_file.lint')
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_roller.py
+++ b/python/mozlint/test/test_roller.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 import sys
 
+import mozunit
 import pytest
 
 from mozlint import ResultContainer
 from mozlint.errors import LintersNotConfigured, LintException
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -74,9 +75,9 @@ def test_roll_with_failure_code(lint, li
 
     assert lint.failed is None
     result = lint.roll(files)
     assert len(result) == 0
     assert lint.failed == ['BadReturnCodeLinter']
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_types.py
+++ b/python/mozlint/test/test_types.py
@@ -1,15 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import sys
 
+import mozunit
 import pytest
 
 from mozlint.result import ResultContainer
 
 
 @pytest.fixture
 def path(filedir):
     def _path(name):
@@ -46,9 +46,9 @@ def test_no_filter(lint, lintdir, files)
     assert len(result) == 0
 
     lint.lintargs['use_filters'] = False
     result = lint.roll(files)
     assert len(result) == 2
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_vcs.py
+++ b/python/mozlint/test/test_vcs.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 import subprocess
-import sys
 
+import mozunit
 import pytest
 
 from mozlint.vcs import VCSHelper, vcs_class
 
 
 setup = {
     'hg': [
         """
@@ -114,9 +114,9 @@ def test_vcs_helper(repo):
 
     assert_files(vcs.by_workdir('all'), [])
     assert_files(vcs.by_workdir('staged'), [])
     assert_files(vcs.by_outgoing(), ['bar', 'baz'])
     assert_files(vcs.by_outgoing(remotepath), ['bar', 'baz'])
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/pytest.ini
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/pytest.ini
@@ -1,7 +1,2 @@
 [pytest]
-# Early-load pytest_mozlog plugin to replace terminal reporter.
-# Adding pytest_mozlog plugin to conftest.py registers the plugin
-# too late for tests to recognize mozlog options.
-# This manual registration of plugin is needed for running these
-# tests in mach, whose virtualenv setup does not call mozlog's setup.py
-addopts = -p mozlog.pytest_mozlog.plugin -p no:terminalreporter
+addopts = -p no:terminalreporter
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import types
 import urllib2
 
+import mozunit
 import pytest
 
 from wptserve.handlers import json_handler
 
 from marionette_harness.runner import httpd
 
 here = os.path.abspath(os.path.dirname(__file__))
 parent = os.path.dirname(here)
@@ -81,11 +82,9 @@ def test_handler(server):
 
     url = server.get_url("/httpd/test_handler")
     body = urllib2.urlopen(url).read()
     res = json.loads(body)
     assert res["count"] == counter
 
 
 if __name__ == "__main__":
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
@@ -1,11 +1,12 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import mozunit
 import pytest
 
 from marionette_harness.runtests import MarionetteArguments
 
 
 @pytest.mark.parametrize("socket_timeout", ['A', '10', '1B-', '1C2', '44.35'])
 def test_parse_arg_socket_timeout(socket_timeout):
     argv = ['marionette', '--socket-timeout', socket_timeout]
@@ -23,11 +24,9 @@ def test_parse_arg_socket_timeout(socket
             parser.parse_args(args=argv)
         assert ex.value.code == 2
     else:
         args = parser.parse_args(args=argv)
         assert hasattr(args, 'socket_timeout') and args.socket_timeout == float(socket_timeout)
 
 
 if __name__ == '__main__':
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
@@ -1,12 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import mozunit
 import pytest
 
 from mock import Mock, patch, sentinel
 
 import marionette_harness.marionette_test as marionette_test
 
 from marionette_harness.runtests import MarionetteTestRunner, MarionetteHarness, cli
 
@@ -99,11 +100,9 @@ def test_harness_sets_up_default_test_ha
     """
     harness = MarionetteHarness(args=mach_parsed_kwargs)
     mach_parsed_kwargs.pop('tests')
     runner = harness._runner_class(**mach_parsed_kwargs)
     assert marionette_test.MarionetteTestCase in runner.test_handlers
 
 
 if __name__ == '__main__':
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -1,13 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import manifestparser
+import mozunit
 import pytest
 
 from mock import Mock, patch, mock_open, sentinel, DEFAULT
 
 from marionette_harness.runtests import MarionetteTestRunner
 
 
 @pytest.fixture
@@ -502,11 +503,9 @@ def test_option_run_until_failure(mach_p
         assert runner.run_until_failure == run_until_failure
         if repeat is None:
             assert runner.repeat == 30
         else:
             assert runner.repeat == repeat
 
 
 if __name__ == '__main__':
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
@@ -1,12 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import mozunit
 import pytest
 
 from marionette_harness import MarionetteTestResult
 
 
 @pytest.fixture
 def empty_marionette_testcase():
     """ Testable MarionetteTestCase class """
@@ -45,11 +46,9 @@ def test_crash_is_recorded_as_error(empt
     assert result.shouldStop == has_crashed
     if has_crashed:
         assert len(result.errors) == 1
     else:
         assert len(result.errors) == 0
 
 
 if __name__ == '__main__':
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import types
 
+import mozunit
 import pytest
 
 from marionette_harness.runner import serve
 from marionette_harness.runner.serve import iter_proc, iter_url
 
 
 def teardown_function(func):
     for server in [server for server in iter_proc(serve.servers) if server.is_alive]:
@@ -58,11 +59,9 @@ def test_iter_url():
 
 def test_where_is():
     serve.start()
     assert serve.where_is("/") == serve.servers["http"][1].get_url("/")
     assert serve.where_is("/", on="https") == serve.servers["https"][1].get_url("/")
 
 
 if __name__ == "__main__":
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/mochitest/tests/python/test_basic_mochitest_plain.py
+++ b/testing/mochitest/tests/python/test_basic_mochitest_plain.py
@@ -1,16 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import sys
 
+import mozunit
 import pytest
 
 from conftest import build, filter_action
 
 sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
 from mozharness.base.log import INFO, WARNING, ERROR
 from mozharness.base.errors import BaseErrorList
 from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
@@ -157,9 +158,9 @@ def test_output_leak(monkeypatch, runtes
 
     errors = filter_action('log', lines)
     errors = [e for e in errors if e['level'] == 'ERROR']
     assert len(errors) == 1
     assert 'leakcheck' in errors[0]['message']
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/testing/mochitest/tests/python/test_get_active_tests.py
+++ b/testing/mochitest/tests/python/test_get_active_tests.py
@@ -1,20 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
 
 import os
-import sys
 from argparse import Namespace
 
 from manifestparser import TestManifest
 
+import mozunit
 import pytest
 
 
 @pytest.fixture
 def get_active_tests(setup_harness_root, parser):
     runtests = pytest.importorskip('runtests')
     md = runtests.MochitestDesktop('plain', {'log_tbpl': '-'})
 
@@ -83,9 +83,9 @@ prefs=
 prefs=foo=bar
 [files/test_fail.html]
 """)[1]
     with pytest.raises(SystemExit):
         get_active_tests(**options)
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/testing/mozbase/mozcrash/tests/test.py
+++ b/testing/mozbase/mozcrash/tests/test.py
@@ -15,17 +15,20 @@ import StringIO
 
 import mozunit
 
 import mozcrash
 import mozhttpd
 import mozlog.unstructured as mozlog
 
 # Make logs go away
-log = mozlog.getLogger("mozcrash", handler=mozlog.FileHandler(os.devnull))
+try:
+    log = mozlog.getLogger("mozcrash", handler=mozlog.FileHandler(os.devnull))
+except ValueError:
+    pass
 
 
 def popen_factory(stdouts):
     """
     Generate a class that can mock subprocess.Popen. |stdouts| is an iterable that
     should return an iterable for the stdout of each process in turn.
     """
     class mock_popen(object):
--- a/testing/mozbase/mozlog/tests/test_logger.py
+++ b/testing/mozbase/mozlog/tests/test_logger.py
@@ -228,17 +228,17 @@ class Loggable(mozlog.LoggingMixin):
 class TestLoggingMixin(unittest.TestCase):
     """Tests basic use of LoggingMixin"""
 
     def test_mixin(self):
         loggable = Loggable()
         self.assertTrue(not hasattr(loggable, "_logger"))
         loggable.log(mozlog.INFO, "This will instantiate the logger")
         self.assertTrue(hasattr(loggable, "_logger"))
-        self.assertEqual(loggable._logger.name, "__main__.Loggable")
+        self.assertEqual(loggable._logger.name, "test_logger.Loggable")
 
         self.assertRaises(ValueError, loggable.set_logger,
                           "not a logger")
 
         logger = mozlog.MozLogger('test.mixin')
         handler = ListHandler()
         logger.addHandler(handler)
         loggable.set_logger(logger)
--- a/xpcom/idl-parser/xpidl/runtests.py
+++ b/xpcom/idl-parser/xpidl/runtests.py
@@ -105,9 +105,9 @@ void getBar();
             def write(self, s):
                 pass
         try:
             header.print_header(i, FdMock(), filename='f')
         except Exception as e:
             self.assertEqual(e.args[0], "Unexpected overloaded virtual method GetBar in interface foo")
 
 if __name__ == '__main__':
-    mozunit.main()
+    mozunit.main(runwith='unittest')