Bug 1261412 - Add mach python-test option to collect tests based on path alone; r=gps
authorMaja Frydrychowicz <mjzffr@gmail.com>
Mon, 18 Apr 2016 10:21:56 -0400
changeset 331517 d06c65a394490475f4d143682deb933baa23d816
parent 331516 b79810219014164bdad95abc806b66a5dd006ec3
child 331518 cf48c9ea48e3ee27b0bb1597a768f4c1fc131c75
push id6048
push userkmoir@mozilla.com
push dateMon, 06 Jun 2016 19:02:08 +0000
treeherdermozilla-beta@46d72a56c57d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1261412
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1261412 - Add mach python-test option to collect tests based on path alone; r=gps MozReview-Commit-ID: 4rsG6sMPqpv
python/mach_commands.py
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -63,56 +63,93 @@ class MachCommands(MachCommandBase):
 
         return self.run_process([self.virtualenv_manager.python_path] + args,
             pass_thru=True,  # Allow user to run Python interactively.
             ensure_exit_code=False,  # Don't throw on non-zero exit code.
             # Note: subprocess requires native strings in os.environ on Windows
             append_env={b'PYTHONDONTWRITEBYTECODE': str('1')})
 
     @Command('python-test', category='testing',
-        description='Run Python unit tests.')
+        description='Run Python unit tests with an appropriate test runner.')
     @CommandArgument('--verbose',
         default=False,
         action='store_true',
         help='Verbose output.')
     @CommandArgument('--stop',
         default=False,
         action='store_true',
         help='Stop running tests after the first error or failure.')
+    @CommandArgument('--path-only',
+        default=False,
+        action='store_true',
+        help=('Collect all tests under given path instead of default '
+              'test resolution. Supports pytest-style tests.'))
     @CommandArgument('tests', nargs='*',
         metavar='TEST',
         help=('Tests to run. Each test can be a single file or a directory. '
-              'Tests must be part of PYTHON_UNIT_TESTS.'))
+              'Default test resolution relies on PYTHON_UNIT_TESTS.'))
     def python_test(self,
                     tests=[],
                     test_objects=None,
                     subsuite=None,
                     verbose=False,
+                    path_only=False,
                     stop=False):
         self._activate_virtualenv()
 
+        def find_tests_by_path():
+            import glob
+            files = []
+            for t in tests:
+                if t.endswith('.py') and os.path.isfile(t):
+                    files.append(t)
+                elif os.path.isdir(t):
+                    for root, _, _ in os.walk(t):
+                        files += glob.glob(mozpath.join(root, 'test*.py'))
+                        files += glob.glob(mozpath.join(root, 'unit*.py'))
+                else:
+                    self.log(logging.WARN, 'python-test',
+                                 {'test': t},
+                                 'TEST-UNEXPECTED-FAIL | Invalid test: {test}')
+                    if stop:
+                        break
+            return files
+
         # Python's unittest, and in particular discover, has problems with
         # clashing namespaces when importing multiple test modules. What follows
         # is a simple way to keep environments separate, at the price of
-        # launching Python multiple times. This also runs tests via mozunit,
+        # launching Python multiple times. Most tests are run via mozunit,
         # which produces output in the format Mozilla infrastructure expects.
+        # Some tests are run via pytest, and these should be equipped with a
+        # local mozunit_report plugin to meet output expectations.
         return_code = 0
         found_tests = False
         if test_objects is None:
             # If we're not being called from `mach test`, do our own
             # test resolution.
-            from mozbuild.testing import TestResolver
-            resolver = self._spawn(TestResolver)
-            if tests:
-                # If we were given test paths, try to find tests matching them.
-                test_objects = resolver.resolve_tests(paths=tests,
-                                                      flavor='python')
+            if path_only:
+                if tests:
+                    self.virtualenv_manager.install_pip_package(
+                       'pytest==2.9.1'
+                    )
+                    test_objects = [{'path': p} for p in find_tests_by_path()]
+                else:
+                    self.log(logging.WARN, 'python-test', {},
+                             'TEST-UNEXPECTED-FAIL | No tests specified')
+                    test_objects = []
             else:
-                # Otherwise just run all Python tests.
-                test_objects = resolver.resolve_tests(flavor='python')
+                from mozbuild.testing import TestResolver
+                resolver = self._spawn(TestResolver)
+                if tests:
+                    # If we were given test paths, try to find tests matching them.
+                    test_objects = resolver.resolve_tests(paths=tests,
+                                                          flavor='python')
+                else:
+                    # Otherwise just run everything in PYTHON_UNIT_TESTS
+                    test_objects = resolver.resolve_tests(flavor='python')
 
         for test in test_objects:
             found_tests = True
             f = test['path']
             file_displayed_test = []  # Used as a boolean.
 
             def _line_handler(line):
                 if not file_displayed_test:
@@ -140,20 +177,20 @@ class MachCommands(MachCommandBase):
                              'Test failed: {file}')
                 else:
                     self.log(logging.INFO, 'python-test', {'file': f},
                              'Test passed: {file}')
             if stop and return_code > 0:
                 return 1
 
         if not found_tests:
-            self.log(logging.WARN, 'python-test', {},
-                     'TEST-UNEXPECTED-FAIL | No tests collected '
-                     '(not in PYTHON_UNIT_TESTS?)')
-
+            message = 'TEST-UNEXPECTED-FAIL | No tests collected'
+            if not path_only:
+                 message += ' (Not in PYTHON_UNIT_TESTS? Try --path-only?)'
+            self.log(logging.WARN, 'python-test', {}, message)
             return 1
 
         return 0 if return_code == 0 else 1
 
     @Command('eslint', category='devenv',
         description='Run eslint or help configure eslint for optimal development.')
     @CommandArgument('-s', '--setup', default=False, action='store_true',
         help='configure eslint for optimal development.')