Bug 1335873 - Convert marionette harness unittests to standard python unittests draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 01 Feb 2017 15:38:42 -0500
changeset 469679 6f8520a811b268b4363160c08875a94a40b68502
parent 469678 370ffae4f59631a8388715b96f4472216027c493
child 544264 110210f49a7db71d7575780108c3df4606641a47
push id43797
push userahalberstadt@mozilla.com
push dateThu, 02 Feb 2017 15:12:36 +0000
bugs1335873
milestone54.0a1
Bug 1335873 - Convert marionette harness unittests to standard python unittests MozReview-Commit-ID: Ata99evHxbd
moz.build
python/mach_commands.py
taskcluster/ci/marionette-harness/kind.yml
taskcluster/ci/source-check/python-tests.yml
taskcluster/docs/kinds.rst
taskcluster/scripts/tester/harness-test-linux.sh
testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini
testing/marionette/harness/marionette_harness/tests/unit/test_marionette.py
testing/mozharness/scripts/marionette_harness_tests.py
--- a/moz.build
+++ b/moz.build
@@ -82,9 +82,14 @@ else:
     TEST_DIRS += ['js/src/tests']
 
 if not CONFIG['JS_STANDALONE'] and CONFIG['MOZ_BUILD_APP']:
     # Bring in the configuration for the configured application.
     include('/' + CONFIG['MOZ_BUILD_APP'] + '/app.mozbuild')
 
 CONFIGURE_SUBST_FILES += ['.cargo/config']
 
+# These python manifests are included here so they get picked up without an objdir
+PYTHON_UNITTEST_MANIFESTS += [
+    'testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini',
+]
+
 include('build/templates.mozbuild')
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -52,21 +52,16 @@ class MachCommands(MachCommandBase):
     @CommandArgument('--verbose',
         default=False,
         action='store_true',
         help='Verbose output.')
     @CommandArgument('--stop',
         default=False,
         action='store_true',
         help='Stop running tests after the first error or failure.')
-    @CommandArgument('--path-only',
-        default=False,
-        action='store_true',
-        help=('Collect all tests under given path instead of default '
-              'test resolution. Supports pytest-style tests.'))
     @CommandArgument('-j', '--jobs',
         default=1,
         type=int,
         help='Number of concurrent jobs to run. Default is 1.')
     @CommandArgument('--subsuite',
         default=None,
         help=('Python subsuite to run. If not specified, all subsuites are run. '
              'Use the string `default` to only run tests without a subsuite.'))
@@ -74,17 +69,16 @@ class MachCommands(MachCommandBase):
         metavar='TEST',
         help=('Tests to run. Each test can be a single file or a directory. '
               'Default test resolution relies on PYTHON_UNITTEST_MANIFESTS.'))
     def python_test(self,
                     tests=[],
                     test_objects=None,
                     subsuite=None,
                     verbose=False,
-                    path_only=False,
                     stop=False,
                     jobs=1):
         self._activate_virtualenv()
 
         def find_tests_by_path():
             import glob
             files = []
             for t in tests:
@@ -104,40 +98,29 @@ class MachCommands(MachCommandBase):
 
         # Python's unittest, and in particular discover, has problems with
         # clashing namespaces when importing multiple test modules. What follows
         # is a simple way to keep environments separate, at the price of
         # launching Python multiple times. Most tests are run via mozunit,
         # which produces output in the format Mozilla infrastructure expects.
         # Some tests are run via pytest.
         if test_objects is None:
-            # If we're not being called from `mach test`, do our own
-            # test resolution.
-            if path_only:
-                if tests:
-                    test_objects = [{'path': p} for p in find_tests_by_path()]
-                else:
-                    self.log(logging.WARN, 'python-test', {},
-                             'TEST-UNEXPECTED-FAIL | No tests specified')
-                    test_objects = []
+            from mozbuild.testing import TestResolver
+            resolver = self._spawn(TestResolver)
+            if tests:
+                # If we were given test paths, try to find tests matching them.
+                test_objects = resolver.resolve_tests(paths=tests,
+                                                      flavor='python')
             else:
-                from mozbuild.testing import TestResolver
-                resolver = self._spawn(TestResolver)
-                if tests:
-                    # If we were given test paths, try to find tests matching them.
-                    test_objects = resolver.resolve_tests(paths=tests,
-                                                          flavor='python')
-                else:
-                    # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
-                    test_objects = resolver.resolve_tests(flavor='python')
+                # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
+                test_objects = resolver.resolve_tests(flavor='python')
 
         if not test_objects:
-            message = 'TEST-UNEXPECTED-FAIL | No tests collected'
-            if not path_only:
-                message += ' (Not in PYTHON_UNITTEST_MANIFESTS? Try --path-only?)'
+            message = 'TEST-UNEXPECTED-FAIL | No tests collected ' + \
+                      '(Not in PYTHON_UNITTEST_MANIFESTS?)'
             self.log(logging.WARN, 'python-test', {}, message)
             return 1
 
         mp = TestManifest()
         mp.tests.extend(test_objects)
 
         filters = []
         if subsuite == 'default':
deleted file mode 100644
--- a/taskcluster/ci/marionette-harness/kind.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst
-
-implementation: taskgraph.task.transform:TransformTask
-
-transforms:
-   - taskgraph.transforms.marionette_harness:transforms
-   - taskgraph.transforms.task:transforms
-
-# NOTE: this task should be refactored so that it is invoked as a job either
-# with a run.using of "mozharness", and combined with the source-check kind.
-
-jobs:
-    marionette-harness/opt:
-        description: "Marionette harness unit test"
-        attributes:
-            build_platform: marionette-harness
-            build_type: opt
-        treeherder:
-            platform: linux64/opt
-            kind: test
-            tier: 2
-            symbol: tc(Mn-h)
-        worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
-        worker:
-            implementation: docker-worker
-            docker-image: {in-tree: desktop-build}  # NOTE: better to use the lint image
-            env:
-                JOB_SCRIPT: "taskcluster/scripts/tester/harness-test-linux.sh"
-                MOZHARNESS_SCRIPT: "testing/mozharness/scripts/marionette_harness_tests.py"
-                TOOLS_DISABLE: "true"
-            artifacts:
-              - name: public/logs/
-                path: /home/worker/workspace/mozharness_workspace/upload/logs/
-                type: directory
-            command:
-              - "bash"
-              - "/home/worker/bin/build.sh"
-              - "--tests=testing/marionette/harness/marionette_harness/tests/harness_unit"
-              - "--work-dir=mozharness_workspace"
-            max-run-time: 1800
-        when:
-            files-changed:
-              - "testing/marionette/harness/**"
-              - "testing/mozharness/scripts/marionette_harness_tests.py"
-              - "testing/config/marionette_harness_test_requirements.txt"
--- a/taskcluster/ci/source-check/python-tests.yml
+++ b/taskcluster/ci/source-check/python-tests.yml
@@ -1,31 +1,32 @@
-taskgraph-tests/opt:
-    description: taskcluster/taskgraph unit tests
+marionette-harness/opt:
+    description: marionette harness unit tests
     treeherder:
-        symbol: py(tg)
+        symbol: py(Mnh)
         kind: test
         tier: 2
         platform: linux64/opt
-    worker-type: aws-provisioner-v1/b2gtest
+    worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
     worker:
         implementation: docker-worker
         docker-image: {in-tree: "lint"}
         max-run-time: 1800
     run:
         using: mach
-        mach: taskgraph python-tests
+        mach: python-test --subsuite marionette-harness
     run-on-projects:
         - integration
         - release
     when:
         files-changed:
-            - 'taskcluster/**/*.py'
-            - 'config/mozunit.py'
-            - 'python/mach/**/*.py'
+          - 'testing/marionette/harness/**'
+          - 'testing/config/marionette_harness_test_requirements.txt'
+          - 'config/mozunit.py'
+          - 'python/mach_commands.py'
 
 mozbase/opt:
     description: testing/mozbase unit tests
     platforms:
         - linux64/opt
     treeherder:
         symbol: py(mb)
         kind: test
@@ -71,8 +72,33 @@ mozharness/opt:
             /usr/bin/pip2 install tox &&
             /home/worker/.local/bin/tox -e py27-hg3.7
     run-on-projects:
         - integration
         - release
     when:
         files-changed:
             - 'testing/mozharness/**'
+
+
+taskgraph-tests/opt:
+    description: taskcluster/taskgraph unit tests
+    treeherder:
+        symbol: tg
+        kind: test
+        tier: 2
+        platform: linux64/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: taskgraph python-tests
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - 'taskcluster/**/*.py'
+            - 'config/mozunit.py'
+            - 'python/mach/**/*.py'
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -85,21 +85,16 @@ will eventually be dependencies of the b
 are run manually via try pushes and the results uploaded to tooltool.
 
 spidermonkey
 ------------
 
 Spidermonkey tasks check out the full gecko source tree, then compile only the
 spidermonkey portion.  Each task runs specific tests after the build.
 
-marionette-harness
-------------------
-
-TBD (Maja)
-
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
deleted file mode 100644
--- a/taskcluster/scripts/tester/harness-test-linux.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#! /bin/bash -vex
-
-set -x -e
-
-echo "running as" $(id)
-
-####
-# Taskcluster friendly wrapper for running a script in
-# testing/mozharness/scripts in a source checkout (no build).
-# Example use: Python-only harness unit tests
-####
-
-: WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
-: SRC_ROOT                      ${SRC_ROOT:=$WORKSPACE/build/src}
-# These paths should be relative to $SRC_ROOT
-: MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
-: MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
-: mozharness args               "${@}"
-
-set -v
-cd $WORKSPACE
-
-fail() {
-    echo # make sure error message is on a new line
-    echo "[harness-test-linux.sh:error]" "${@}"
-    exit 1
-}
-
-if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
-
-# support multiple, space delimited, config files
-config_cmds=""
-for cfg in $MOZHARNESS_CONFIG; do
-  config_cmds="${config_cmds} --config-file ${SRC_ROOT}/${cfg}"
-done
-
-python2.7 $SRC_ROOT/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}"
-
-
-
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini
@@ -0,0 +1,9 @@
+[DEFAULT]
+subsuite = marionette-harness
+
+[test_httpd.py]
+[test_marionette_arguments.py]
+[test_marionette_harness.py]
+[test_marionette_runner.py]
+[test_marionette_test_result.py]
+[test_serve.py]
--- a/testing/marionette/harness/marionette_harness/tests/unit/test_marionette.py
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_marionette.py
@@ -1,15 +1,18 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import itertools
+import sys
 import time
 
+import pytest
+
 from marionette_driver import errors
 
 from marionette_harness import MarionetteTestCase, run_if_manage_instance, skip_if_mobile
 
 
 class TestMarionette(MarionetteTestCase):
 
     def test_correct_test_name(self):
@@ -60,8 +63,12 @@ class TestProtocol2Errors(MarionetteTest
                  "stacktrace": None})
 
     def test_unknown_error_status(self):
         with self.assertRaises(errors.MarionetteException):
             self.marionette._handle_error(
                 {"error": "barbera",
                  "message": None,
                  "stacktrace": None})
+
+
+if __name__ == '__main__':
+    sys.exit(pytest.main(['-p', 'no:terminalreporter', '--log-tbpl', '-', __file__]))
deleted file mode 100644
--- a/testing/mozharness/scripts/marionette_harness_tests.py
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-import copy
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.python import PreScriptAction
-from mozharness.base.python import (
-    VirtualenvMixin,
-    virtualenv_config_options,
-)
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.buildbot import (
-    BuildbotMixin, TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE,
-    TBPL_EXCEPTION
-)
-
-marionette_harness_tests_config_options = [
-    [['--tests'], {
-        'dest': 'test_path',
-        'default': None,
-        'help': 'Path to test_*.py or directory relative to src root.',
-    }],
-    [['--src-dir'], {
-        'dest': 'rel_src_dir',
-        'default': None,
-        'help': 'Path to hg.mo source checkout relative to work dir.',
-    }],
-
-] + copy.deepcopy(virtualenv_config_options)
-
-marionette_harness_tests_config = {
-    "find_links": [
-        "http://pypi.pub.build.mozilla.org/pub",
-    ],
-    "pip_index": False,
-    # relative to workspace
-    "rel_src_dir": os.path.join("build", "src"),
-}
-
-class MarionetteHarnessTests(VirtualenvMixin, BuildbotMixin, BaseScript):
-
-    def __init__(self, config_options=None,
-                 all_actions=None, default_actions=None,
-                 *args, **kwargs):
-        config_options = config_options or marionette_harness_tests_config_options
-        actions = [
-            'clobber',
-            'create-virtualenv',
-            'run-tests',
-        ]
-        super(MarionetteHarnessTests, self).__init__(
-            config_options=config_options,
-            all_actions=all_actions or actions,
-            default_actions=default_actions or actions,
-            config=marionette_harness_tests_config,
-            *args, **kwargs)
-
-    @PreScriptAction('create-virtualenv')
-    def _pre_create_virtualenv(self, action):
-        dirs = self.query_abs_dirs()
-        c = self.config
-        requirements = os.path.join(
-            dirs['abs_src_dir'],
-            'testing', 'config',
-            'marionette_harness_test_requirements.txt'
-        )
-        self.register_virtualenv_module(
-           requirements=[requirements],
-           two_pass=True
-        )
-
-    def query_abs_dirs(self):
-        if self.abs_dirs:
-            return self.abs_dirs
-        c = self.config
-        abs_dirs = super(MarionetteHarnessTests, self).query_abs_dirs()
-        dirs = {
-            'abs_src_dir': os.path.abspath(
-                os.path.join(abs_dirs['base_work_dir'], c['rel_src_dir'])
-            ),
-        }
-
-        for key in dirs:
-            if key not in abs_dirs:
-                abs_dirs[key] = dirs[key]
-        self.abs_dirs = abs_dirs
-
-        return self.abs_dirs
-
-    def _get_pytest_status(self, code):
-        """
-        Translate pytest exit code to TH status
-
-        Based on https://github.com/pytest-dev/pytest/blob/master/_pytest/main.py#L21-L26
-        """
-        if code == 0:
-            return TBPL_SUCCESS
-        elif code == 1:
-            return TBPL_WARNING
-        elif 1 < code < 6:
-            self.error("pytest returned exit code: %s" % code)
-            return TBPL_FAILURE
-        else:
-            return TBPL_EXCEPTION
-
-    def run_tests(self):
-        """Run all the tests"""
-        dirs = self.query_abs_dirs()
-        test_relpath = self.config.get(
-            'test_path',
-            os.path.join('testing', 'marionette',
-                         'harness', 'marionette_harness', 'tests',
-                         'harness_unit')
-        )
-        test_path = os.path.join(dirs['abs_src_dir'], test_relpath)
-        self.activate_virtualenv()
-        import pytest
-        command = ['-p', 'no:terminalreporter',  # disable pytest logging
-                   test_path]
-        logs = {}
-        for fmt in ['tbpl', 'mach', 'raw']:
-            logs[fmt] = os.path.join(dirs['abs_log_dir'],
-                                     'mn-harness_{}.log'.format(fmt))
-            command.extend(['--log-'+fmt, logs[fmt]])
-        self.info('Calling pytest.main with the following arguments: %s' % command)
-        status = self._get_pytest_status(pytest.main(command))
-        self.read_from_file(logs['tbpl'])
-        for log in logs.values():
-            self.copy_to_upload_dir(log, dest='logs/')
-        self.buildbot_status(status)
-
-
-if __name__ == '__main__':
-    script = MarionetteHarnessTests()
-    script.run_and_exit()