Backed out changeset 3c0a7527608a (bug 1335873)
authorSebastian Hengst <archaeopteryx@coole-files.de>
Wed, 01 Mar 2017 17:35:06 +0100
changeset 374439 29f59f3f6882445a6782a59ab55fe9473555f2ce
parent 374438 d2b0ed8f69e0c58328b772d235d4202103af3497
child 374440 d725ecdbe54e6f3e39f86cc99f8e638d6b614431
push id10863
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 23:02:23 +0000
treeherdermozilla-aurora@0931190cd725 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1335873
milestone54.0a1
backs out3c0a7527608aa6398f8177b08088da39592078d9
Backed out changeset 3c0a7527608a (bug 1335873)
python/mach_commands.py
taskcluster/ci/marionette-harness/kind.yml
taskcluster/ci/source-test/python-tests.yml
taskcluster/docs/kinds.rst
taskcluster/scripts/tester/harness-test-linux.sh
taskcluster/taskgraph/try_option_syntax.py
testing/config/marionette_harness_test_requirements.txt
testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini
testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
testing/mozbase/packages.txt
testing/mozharness/scripts/marionette_harness_tests.py
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -52,16 +52,21 @@ class MachCommands(MachCommandBase):
     @CommandArgument('--verbose',
         default=False,
         action='store_true',
         help='Verbose output.')
     @CommandArgument('--stop',
         default=False,
         action='store_true',
         help='Stop running tests after the first error or failure.')
+    @CommandArgument('--path-only',
+        default=False,
+        action='store_true',
+        help=('Collect all tests under given path instead of default '
+              'test resolution. Supports pytest-style tests.'))
     @CommandArgument('-j', '--jobs',
         default=1,
         type=int,
         help='Number of concurrent jobs to run. Default is 1.')
     @CommandArgument('--subsuite',
         default=None,
         help=('Python subsuite to run. If not specified, all subsuites are run. '
              'Use the string `default` to only run tests without a subsuite.'))
@@ -69,16 +74,17 @@ class MachCommands(MachCommandBase):
         metavar='TEST',
         help=('Tests to run. Each test can be a single file or a directory. '
               'Default test resolution relies on PYTHON_UNITTEST_MANIFESTS.'))
     def python_test(self,
                     tests=[],
                     test_objects=None,
                     subsuite=None,
                     verbose=False,
+                    path_only=False,
                     stop=False,
                     jobs=1):
         self._activate_virtualenv()
 
         def find_tests_by_path():
             import glob
             files = []
             for t in tests:
@@ -98,29 +104,40 @@ class MachCommands(MachCommandBase):
 
         # Python's unittest, and in particular discover, has problems with
         # clashing namespaces when importing multiple test modules. What follows
         # is a simple way to keep environments separate, at the price of
         # launching Python multiple times. Most tests are run via mozunit,
         # which produces output in the format Mozilla infrastructure expects.
         # Some tests are run via pytest.
         if test_objects is None:
-            from mozbuild.testing import TestResolver
-            resolver = self._spawn(TestResolver)
-            if tests:
-                # If we were given test paths, try to find tests matching them.
-                test_objects = resolver.resolve_tests(paths=tests,
-                                                      flavor='python')
+            # If we're not being called from `mach test`, do our own
+            # test resolution.
+            if path_only:
+                if tests:
+                    test_objects = [{'path': p} for p in find_tests_by_path()]
+                else:
+                    self.log(logging.WARN, 'python-test', {},
+                             'TEST-UNEXPECTED-FAIL | No tests specified')
+                    test_objects = []
             else:
-                # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
-                test_objects = resolver.resolve_tests(flavor='python')
+                from mozbuild.testing import TestResolver
+                resolver = self._spawn(TestResolver)
+                if tests:
+                    # If we were given test paths, try to find tests matching them.
+                    test_objects = resolver.resolve_tests(paths=tests,
+                                                          flavor='python')
+                else:
+                    # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
+                    test_objects = resolver.resolve_tests(flavor='python')
 
         if not test_objects:
-            message = 'TEST-UNEXPECTED-FAIL | No tests collected ' + \
-                      '(Not in PYTHON_UNITTEST_MANIFESTS?)'
+            message = 'TEST-UNEXPECTED-FAIL | No tests collected'
+            if not path_only:
+                message += ' (Not in PYTHON_UNITTEST_MANIFESTS? Try --path-only?)'
             self.log(logging.WARN, 'python-test', {}, message)
             return 1
 
         mp = TestManifest()
         mp.tests.extend(test_objects)
 
         filters = []
         if subsuite == 'default':
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/marionette-harness/kind.yml
@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.marionette_harness:transforms
+   - taskgraph.transforms.task:transforms
+
+# NOTE: this task should be refactored so that it is invoked as a job either
+# with a run.using of "mozharness", and combined with the source-check kind.
+
+jobs:
+    marionette-harness/opt:
+        description: "Marionette harness unit test"
+        attributes:
+            build_platform: marionette-harness
+            build_type: opt
+        treeherder:
+            platform: linux64/opt
+            kind: test
+            tier: 2
+            symbol: tc(Mn-h)
+        worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+        worker:
+            implementation: docker-worker
+            docker-image: {in-tree: desktop-build}  # NOTE: better to use the lint image
+            env:
+                JOB_SCRIPT: "taskcluster/scripts/tester/harness-test-linux.sh"
+                MOZHARNESS_SCRIPT: "testing/mozharness/scripts/marionette_harness_tests.py"
+                TOOLS_DISABLE: "true"
+            artifacts:
+              - name: public/logs/
+                path: /home/worker/workspace/mozharness_workspace/upload/logs/
+                type: directory
+            command:
+              - "bash"
+              - "/home/worker/bin/build.sh"
+              - "--tests=testing/marionette/harness/marionette_harness/tests/harness_unit"
+              - "--work-dir=mozharness_workspace"
+            max-run-time: 1800
+        when:
+            files-changed:
+              - "testing/marionette/harness/**"
+              - "testing/mozbase/mozlog/mozlog/pytest_mozlog/**"
+              - "testing/mozharness/scripts/marionette_harness_tests.py"
+              - "testing/config/marionette_harness_test_requirements.txt"
--- a/taskcluster/ci/source-test/python-tests.yml
+++ b/taskcluster/ci/source-test/python-tests.yml
@@ -17,45 +17,16 @@ taskgraph-tests/opt:
         - integration
         - release
     when:
         files-changed:
             - 'taskcluster/**/*.py'
             - 'config/mozunit.py'
             - 'python/mach/**/*.py'
 
-marionette-harness/opt:
-    description: testing/marionette/harness unit tests
-    platforms:
-        - linux64/opt
-    treeherder:
-        symbol: py(mnh)
-        kind: test
-        tier: 2
-    worker-type:
-        by-platform:
-            linux64.*: aws-provisioner-v1/b2gtest
-    worker:
-        by-platform:
-            linux64.*:
-                implementation: docker-worker
-                docker-image: {in-tree: "lint"}
-                max-run-time: 3600
-    run:
-        using: mach
-        mach: python-test --subsuite marionette-harness
-    run-on-projects:
-        - integration
-        - release
-    when:
-        files-changed:
-          - 'testing/marionette/harness/**'
-          - 'testing/mozbase/mozlog/mozlog/pytest_mozlog/**'
-          - 'python/mach_commands.py'
-
 mozbase/opt:
     description: testing/mozbase unit tests
     platforms:
         - linux64/opt
     treeherder:
         symbol: py(mb)
         kind: test
         tier: 2
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -85,16 +85,21 @@ will eventually be dependencies of the b
 are run manually via try pushes and the results uploaded to tooltool.
 
 spidermonkey
 ------------
 
 Spidermonkey tasks check out the full gecko source tree, then compile only the
 spidermonkey portion.  Each task runs specific tests after the build.
 
+marionette-harness
+------------------
+
+TBD (Maja)
+
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
new file mode 100644
--- /dev/null
+++ b/taskcluster/scripts/tester/harness-test-linux.sh
@@ -0,0 +1,40 @@
+#! /bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+####
+# Taskcluster friendly wrapper for running a script in
+# testing/mozharness/scripts in a source checkout (no build).
+# Example use: Python-only harness unit tests
+####
+
+: WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
+: SRC_ROOT                      ${SRC_ROOT:=$WORKSPACE/build/src}
+# These paths should be relative to $SRC_ROOT
+: MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
+: MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
+: mozharness args               "${@}"
+
+set -v
+cd $WORKSPACE
+
+fail() {
+    echo # make sure error message is on a new line
+    echo "[harness-test-linux.sh:error]" "${@}"
+    exit 1
+}
+
+if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
+
+# support multiple, space delimited, config files
+config_cmds=""
+for cfg in $MOZHARNESS_CONFIG; do
+  config_cmds="${config_cmds} --config-file ${SRC_ROOT}/${cfg}"
+done
+
+python2.7 $SRC_ROOT/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}"
+
+
+
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -32,16 +32,17 @@ BUILD_KINDS = set([
     'static-analysis',
     'spidermonkey',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
     'source-test',
     'toolchain',
+    'marionette-harness',
     'android-stuff',
 ])
 
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying
 # matching test names
 def alias_prefix(prefix):
     return lambda name: name.startswith(prefix)
new file mode 100644
--- /dev/null
+++ b/testing/config/marionette_harness_test_requirements.txt
@@ -0,0 +1,13 @@
+-r mozbase_requirements.txt
+
+# TODO - if we structure common.tests.zip to match the in-tree structure of the
+# testing directory, we could use ./marionette_requirements.txt instead
+../web-platform/tests/tools/wptserve
+../marionette/client
+../marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
+../marionette/harness
+
+# pytest
+../../python/py
+../../python/pytest
+../../python/mock-1.0.0
deleted file mode 100644
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini
+++ /dev/null
@@ -1,9 +0,0 @@
-[DEFAULT]
-subsuite = marionette-harness
-
-[test_httpd.py]
-[test_marionette_arguments.py]
-[test_marionette_harness.py]
-[test_marionette_runner.py]
-[test_marionette_test_result.py]
-[test_serve.py]
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
@@ -82,10 +82,9 @@ def test_handler(server):
     url = server.get_url("/httpd/test_handler")
     body = urllib2.urlopen(url).read()
     res = json.loads(body)
     assert res["count"] == counter
 
 
 if __name__ == "__main__":
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(["--verbose", __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
@@ -24,10 +24,9 @@ def test_parse_arg_socket_timeout(socket
         assert ex.value.code == 2
     else:
         args = parser.parse_args(args=argv)
         assert hasattr(args, 'socket_timeout') and args.socket_timeout == float(socket_timeout)
 
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
@@ -100,10 +100,9 @@ def test_harness_sets_up_default_test_ha
     harness = MarionetteHarness(args=mach_parsed_kwargs)
     mach_parsed_kwargs.pop('tests')
     runner = harness._runner_class(**mach_parsed_kwargs)
     assert marionette_test.MarionetteTestCase in runner.test_handlers
 
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -434,10 +434,9 @@ def test_e10s_option_sets_prefs(mach_par
 def test_e10s_option_clash_raises(mock_runner):
     mock_runner.e10s = False
     with pytest.raises(AssertionError) as e:
         mock_runner.run_tests([u'test_fake_thing.py'])
         assert "configuration (self.e10s) does not match browser appinfo" in e.value.message
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
@@ -46,10 +46,9 @@ def test_crash_is_recorded_as_error(empt
     if has_crashed:
         assert len(result.errors) == 1
     else:
         assert len(result.errors) == 0
 
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
@@ -59,10 +59,9 @@ def test_iter_url():
 def test_where_is():
     serve.start()
     assert serve.where_is("/") == serve.servers["http"][1].get_url("/")
     assert serve.where_is("/", on="https") == serve.servers["https"][1].get_url("/")
 
 
 if __name__ == "__main__":
     import sys
-    sys.exit(pytest.main(
-        ['-s', '-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(["-s", "--verbose", __file__]))
--- a/testing/mozbase/packages.txt
+++ b/testing/mozbase/packages.txt
@@ -3,17 +3,17 @@ mozb2g.pth:testing/mozbase/mozb2g
 mozcrash.pth:testing/mozbase/mozcrash
 mozdebug.pth:testing/mozbase/mozdebug
 mozdevice.pth:testing/mozbase/mozdevice
 mozfile.pth:testing/mozbase/mozfile
 mozhttpd.pth:testing/mozbase/mozhttpd
 mozinfo.pth:testing/mozbase/mozinfo
 mozinstall.pth:testing/mozbase/mozinstall
 mozleak.pth:testing/mozbase/mozleak
-setup.py:testing/mozbase/mozlog:develop
+mozlog.pth:testing/mozbase/mozlog
 moznetwork.pth:testing/mozbase/moznetwork
 mozprocess.pth:testing/mozbase/mozprocess
 mozprofile.pth:testing/mozbase/mozprofile
 mozrunner.pth:testing/mozbase/mozrunner
 mozsystemmonitor.pth:testing/mozbase/mozsystemmonitor
 mozscreenshot.pth:testing/mozbase/mozscreenshot
 moztest.pth:testing/mozbase/moztest
 mozversion.pth:testing/mozbase/mozversion
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/marionette_harness_tests.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+import copy
+import os
+import sys
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.python import PreScriptAction
+from mozharness.base.python import (
+    VirtualenvMixin,
+    virtualenv_config_options,
+)
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.buildbot import (
+    BuildbotMixin, TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE,
+    TBPL_EXCEPTION
+)
+
+marionette_harness_tests_config_options = [
+    [['--tests'], {
+        'dest': 'test_path',
+        'default': None,
+        'help': 'Path to test_*.py or directory relative to src root.',
+    }],
+    [['--src-dir'], {
+        'dest': 'rel_src_dir',
+        'default': None,
+        'help': 'Path to hg.mo source checkout relative to work dir.',
+    }],
+
+] + copy.deepcopy(virtualenv_config_options)
+
+marionette_harness_tests_config = {
+    "find_links": [
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    # relative to workspace
+    "rel_src_dir": os.path.join("build", "src"),
+}
+
+class MarionetteHarnessTests(VirtualenvMixin, BuildbotMixin, BaseScript):
+
+    def __init__(self, config_options=None,
+                 all_actions=None, default_actions=None,
+                 *args, **kwargs):
+        config_options = config_options or marionette_harness_tests_config_options
+        actions = [
+            'clobber',
+            'create-virtualenv',
+            'run-tests',
+        ]
+        super(MarionetteHarnessTests, self).__init__(
+            config_options=config_options,
+            all_actions=all_actions or actions,
+            default_actions=default_actions or actions,
+            config=marionette_harness_tests_config,
+            *args, **kwargs)
+
+    @PreScriptAction('create-virtualenv')
+    def _pre_create_virtualenv(self, action):
+        dirs = self.query_abs_dirs()
+        c = self.config
+        requirements = os.path.join(
+            dirs['abs_src_dir'],
+            'testing', 'config',
+            'marionette_harness_test_requirements.txt'
+        )
+        self.register_virtualenv_module(
+           requirements=[requirements],
+           two_pass=True
+        )
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        c = self.config
+        abs_dirs = super(MarionetteHarnessTests, self).query_abs_dirs()
+        dirs = {
+            'abs_src_dir': os.path.abspath(
+                os.path.join(abs_dirs['base_work_dir'], c['rel_src_dir'])
+            ),
+        }
+
+        for key in dirs:
+            if key not in abs_dirs:
+                abs_dirs[key] = dirs[key]
+        self.abs_dirs = abs_dirs
+
+        return self.abs_dirs
+
+    def _get_pytest_status(self, code):
+        """
+        Translate pytest exit code to TH status
+
+        Based on https://github.com/pytest-dev/pytest/blob/master/_pytest/main.py#L21-L26
+        """
+        if code == 0:
+            return TBPL_SUCCESS
+        elif code == 1:
+            return TBPL_WARNING
+        elif 1 < code < 6:
+            self.error("pytest returned exit code: %s" % code)
+            return TBPL_FAILURE
+        else:
+            return TBPL_EXCEPTION
+
+    def run_tests(self):
+        """Run all the tests"""
+        dirs = self.query_abs_dirs()
+        test_relpath = self.config.get(
+            'test_path',
+            os.path.join('testing', 'marionette',
+                         'harness', 'marionette_harness', 'tests',
+                         'harness_unit')
+        )
+        test_path = os.path.join(dirs['abs_src_dir'], test_relpath)
+        self.activate_virtualenv()
+        import pytest
+        command = ['-p', 'no:terminalreporter',  # disable pytest logging
+                   test_path]
+        logs = {}
+        for fmt in ['tbpl', 'mach', 'raw']:
+            logs[fmt] = os.path.join(dirs['abs_log_dir'],
+                                     'mn-harness_{}.log'.format(fmt))
+            command.extend(['--log-'+fmt, logs[fmt]])
+        self.info('Calling pytest.main with the following arguments: %s' % command)
+        status = self._get_pytest_status(pytest.main(command))
+        self.read_from_file(logs['tbpl'])
+        for log in logs.values():
+            self.copy_to_upload_dir(log, dest='logs/')
+        self.buildbot_status(status)
+
+
+if __name__ == '__main__':
+    script = MarionetteHarnessTests()
+    script.run_and_exit()