author | Sebastian Hengst <archaeopteryx@coole-files.de> |
Wed, 01 Mar 2017 17:35:06 +0100 | |
changeset 392054 | 29f59f3f6882445a6782a59ab55fe9473555f2ce |
parent 392053 | d2b0ed8f69e0c58328b772d235d4202103af3497 |
child 392055 | d725ecdbe54e6f3e39f86cc99f8e638d6b614431 |
push id | 7198 |
push user | jlorenzo@mozilla.com |
push date | Tue, 18 Apr 2017 12:07:49 +0000 |
treeherder | mozilla-beta@d57aa49c3948 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
bugs | 1335873 |
milestone | 54.0a1 |
backs out | 3c0a7527608aa6398f8177b08088da39592078d9 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/python/mach_commands.py +++ b/python/mach_commands.py @@ -52,16 +52,21 @@ class MachCommands(MachCommandBase): @CommandArgument('--verbose', default=False, action='store_true', help='Verbose output.') @CommandArgument('--stop', default=False, action='store_true', help='Stop running tests after the first error or failure.') + @CommandArgument('--path-only', + default=False, + action='store_true', + help=('Collect all tests under given path instead of default ' + 'test resolution. Supports pytest-style tests.')) @CommandArgument('-j', '--jobs', default=1, type=int, help='Number of concurrent jobs to run. Default is 1.') @CommandArgument('--subsuite', default=None, help=('Python subsuite to run. If not specified, all subsuites are run. ' 'Use the string `default` to only run tests without a subsuite.')) @@ -69,16 +74,17 @@ class MachCommands(MachCommandBase): metavar='TEST', help=('Tests to run. Each test can be a single file or a directory. ' 'Default test resolution relies on PYTHON_UNITTEST_MANIFESTS.')) def python_test(self, tests=[], test_objects=None, subsuite=None, verbose=False, + path_only=False, stop=False, jobs=1): self._activate_virtualenv() def find_tests_by_path(): import glob files = [] for t in tests: @@ -98,29 +104,40 @@ class MachCommands(MachCommandBase): # Python's unittest, and in particular discover, has problems with # clashing namespaces when importing multiple test modules. What follows # is a simple way to keep environments separate, at the price of # launching Python multiple times. Most tests are run via mozunit, # which produces output in the format Mozilla infrastructure expects. # Some tests are run via pytest. if test_objects is None: - from mozbuild.testing import TestResolver - resolver = self._spawn(TestResolver) - if tests: - # If we were given test paths, try to find tests matching them. - test_objects = resolver.resolve_tests(paths=tests, - flavor='python') + # If we're not being called from `mach test`, do our own + # test resolution. + if path_only: + if tests: + test_objects = [{'path': p} for p in find_tests_by_path()] + else: + self.log(logging.WARN, 'python-test', {}, + 'TEST-UNEXPECTED-FAIL | No tests specified') + test_objects = [] else: - # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS - test_objects = resolver.resolve_tests(flavor='python') + from mozbuild.testing import TestResolver + resolver = self._spawn(TestResolver) + if tests: + # If we were given test paths, try to find tests matching them. + test_objects = resolver.resolve_tests(paths=tests, + flavor='python') + else: + # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS + test_objects = resolver.resolve_tests(flavor='python') if not test_objects: - message = 'TEST-UNEXPECTED-FAIL | No tests collected ' + \ - '(Not in PYTHON_UNITTEST_MANIFESTS?)' + message = 'TEST-UNEXPECTED-FAIL | No tests collected' + if not path_only: + message += ' (Not in PYTHON_UNITTEST_MANIFESTS? Try --path-only?)' self.log(logging.WARN, 'python-test', {}, message) return 1 mp = TestManifest() mp.tests.extend(test_objects) filters = [] if subsuite == 'default':
new file mode 100644 --- /dev/null +++ b/taskcluster/ci/marionette-harness/kind.yml @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.marionette_harness:transforms + - taskgraph.transforms.task:transforms + +# NOTE: this task should be refactored so that it is invoked as a job either +# with a run.using of "mozharness", and combined with the source-check kind. + +jobs: + marionette-harness/opt: + description: "Marionette harness unit test" + attributes: + build_platform: marionette-harness + build_type: opt + treeherder: + platform: linux64/opt + kind: test + tier: 2 + symbol: tc(Mn-h) + worker-type: aws-provisioner-v1/gecko-t-linux-xlarge + worker: + implementation: docker-worker + docker-image: {in-tree: desktop-build} # NOTE: better to use the lint image + env: + JOB_SCRIPT: "taskcluster/scripts/tester/harness-test-linux.sh" + MOZHARNESS_SCRIPT: "testing/mozharness/scripts/marionette_harness_tests.py" + TOOLS_DISABLE: "true" + artifacts: + - name: public/logs/ + path: /home/worker/workspace/mozharness_workspace/upload/logs/ + type: directory + command: + - "bash" + - "/home/worker/bin/build.sh" + - "--tests=testing/marionette/harness/marionette_harness/tests/harness_unit" + - "--work-dir=mozharness_workspace" + max-run-time: 1800 + when: + files-changed: + - "testing/marionette/harness/**" + - "testing/mozbase/mozlog/mozlog/pytest_mozlog/**" + - "testing/mozharness/scripts/marionette_harness_tests.py" + - "testing/config/marionette_harness_test_requirements.txt"
--- a/taskcluster/ci/source-test/python-tests.yml +++ b/taskcluster/ci/source-test/python-tests.yml @@ -17,45 +17,16 @@ taskgraph-tests/opt: - integration - release when: files-changed: - 'taskcluster/**/*.py' - 'config/mozunit.py' - 'python/mach/**/*.py' -marionette-harness/opt: - description: testing/marionette/harness unit tests - platforms: - - linux64/opt - treeherder: - symbol: py(mnh) - kind: test - tier: 2 - worker-type: - by-platform: - linux64.*: aws-provisioner-v1/b2gtest - worker: - by-platform: - linux64.*: - implementation: docker-worker - docker-image: {in-tree: "lint"} - max-run-time: 3600 - run: - using: mach - mach: python-test --subsuite marionette-harness - run-on-projects: - - integration - - release - when: - files-changed: - - 'testing/marionette/harness/**' - - 'testing/mozbase/mozlog/mozlog/pytest_mozlog/**' - - 'python/mach_commands.py' - mozbase/opt: description: testing/mozbase unit tests platforms: - linux64/opt treeherder: symbol: py(mb) kind: test tier: 2
--- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -85,16 +85,21 @@ will eventually be dependencies of the b are run manually via try pushes and the results uploaded to tooltool. spidermonkey ------------ Spidermonkey tasks check out the full gecko source tree, then compile only the spidermonkey portion. Each task runs specific tests after the build. +marionette-harness +------------------ + +TBD (Maja) + Tests ----- Test tasks for Gecko products are divided into several kinds, but share a common implementation. The process goes like this, based on a set of YAML files named in ``kind.yml``: * For each build task, determine the related test platforms based on the build
new file mode 100644 --- /dev/null +++ b/taskcluster/scripts/tester/harness-test-linux.sh @@ -0,0 +1,40 @@ +#! /bin/bash -vex + +set -x -e + +echo "running as" $(id) + +#### +# Taskcluster friendly wrapper for running a script in +# testing/mozharness/scripts in a source checkout (no build). +# Example use: Python-only harness unit tests +#### + +: WORKSPACE ${WORKSPACE:=/home/worker/workspace} +: SRC_ROOT ${SRC_ROOT:=$WORKSPACE/build/src} +# These paths should be relative to $SRC_ROOT +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: mozharness args "${@}" + +set -v +cd $WORKSPACE + +fail() { + echo # make sure error message is on a new line + echo "[harness-test-linux.sh:error]" "${@}" + exit 1 +} + +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config-file ${SRC_ROOT}/${cfg}" +done + +python2.7 $SRC_ROOT/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}" + + +
--- a/taskcluster/taskgraph/try_option_syntax.py +++ b/taskcluster/taskgraph/try_option_syntax.py @@ -32,16 +32,17 @@ BUILD_KINDS = set([ 'static-analysis', 'spidermonkey', ]) # anything in this list is governed by -j JOB_KINDS = set([ 'source-test', 'toolchain', + 'marionette-harness', 'android-stuff', ]) # mapping from shortcut name (usable with -u) to a boolean function identifying # matching test names def alias_prefix(prefix): return lambda name: name.startswith(prefix)
new file mode 100644 --- /dev/null +++ b/testing/config/marionette_harness_test_requirements.txt @@ -0,0 +1,13 @@ +-r mozbase_requirements.txt + +# TODO - if we structure common.tests.zip to match the in-tree structure of the +# testing directory, we could use ./marionette_requirements.txt instead +../web-platform/tests/tools/wptserve +../marionette/client +../marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py +../marionette/harness + +# pytest +../../python/py +../../python/pytest +../../python/mock-1.0.0
deleted file mode 100644 --- a/testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini +++ /dev/null @@ -1,9 +0,0 @@ -[DEFAULT] -subsuite = marionette-harness - -[test_httpd.py] -[test_marionette_arguments.py] -[test_marionette_harness.py] -[test_marionette_runner.py] -[test_marionette_test_result.py] -[test_serve.py]
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py @@ -82,10 +82,9 @@ def test_handler(server): url = server.get_url("/httpd/test_handler") body = urllib2.urlopen(url).read() res = json.loads(body) assert res["count"] == counter if __name__ == "__main__": import sys - sys.exit(pytest.main( - ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) + sys.exit(pytest.main(["--verbose", __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py @@ -24,10 +24,9 @@ def test_parse_arg_socket_timeout(socket assert ex.value.code == 2 else: args = parser.parse_args(args=argv) assert hasattr(args, 'socket_timeout') and args.socket_timeout == float(socket_timeout) if __name__ == '__main__': import sys - sys.exit(pytest.main( - ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) + sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py @@ -100,10 +100,9 @@ def test_harness_sets_up_default_test_ha harness = MarionetteHarness(args=mach_parsed_kwargs) mach_parsed_kwargs.pop('tests') runner = harness._runner_class(**mach_parsed_kwargs) assert marionette_test.MarionetteTestCase in runner.test_handlers if __name__ == '__main__': import sys - sys.exit(pytest.main( - ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) + sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py @@ -434,10 +434,9 @@ def test_e10s_option_sets_prefs(mach_par def test_e10s_option_clash_raises(mock_runner): mock_runner.e10s = False with pytest.raises(AssertionError) as e: mock_runner.run_tests([u'test_fake_thing.py']) assert "configuration (self.e10s) does not match browser appinfo" in e.value.message if __name__ == '__main__': import sys - sys.exit(pytest.main( - ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) + sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py @@ -46,10 +46,9 @@ def test_crash_is_recorded_as_error(empt if has_crashed: assert len(result.errors) == 1 else: assert len(result.errors) == 0 if __name__ == '__main__': import sys - sys.exit(pytest.main( - ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) + sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py @@ -59,10 +59,9 @@ def test_iter_url(): def test_where_is(): serve.start() assert serve.where_is("/") == serve.servers["http"][1].get_url("/") assert serve.where_is("/", on="https") == serve.servers["https"][1].get_url("/") if __name__ == "__main__": import sys - sys.exit(pytest.main( - ['-s', '-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) + sys.exit(pytest.main(["-s", "--verbose", __file__]))
--- a/testing/mozbase/packages.txt +++ b/testing/mozbase/packages.txt @@ -3,17 +3,17 @@ mozb2g.pth:testing/mozbase/mozb2g mozcrash.pth:testing/mozbase/mozcrash mozdebug.pth:testing/mozbase/mozdebug mozdevice.pth:testing/mozbase/mozdevice mozfile.pth:testing/mozbase/mozfile mozhttpd.pth:testing/mozbase/mozhttpd mozinfo.pth:testing/mozbase/mozinfo mozinstall.pth:testing/mozbase/mozinstall mozleak.pth:testing/mozbase/mozleak -setup.py:testing/mozbase/mozlog:develop +mozlog.pth:testing/mozbase/mozlog moznetwork.pth:testing/mozbase/moznetwork mozprocess.pth:testing/mozbase/mozprocess mozprofile.pth:testing/mozbase/mozprofile mozrunner.pth:testing/mozbase/mozrunner mozsystemmonitor.pth:testing/mozbase/mozsystemmonitor mozscreenshot.pth:testing/mozbase/mozscreenshot moztest.pth:testing/mozbase/moztest mozversion.pth:testing/mozbase/mozversion
new file mode 100644 --- /dev/null +++ b/testing/mozharness/scripts/marionette_harness_tests.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +import copy +import os +import sys + +# load modules from parent dir +sys.path.insert(1, os.path.dirname(sys.path[0])) + +from mozharness.base.python import PreScriptAction +from mozharness.base.python import ( + VirtualenvMixin, + virtualenv_config_options, +) +from mozharness.base.script import BaseScript +from mozharness.mozilla.buildbot import ( + BuildbotMixin, TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE, + TBPL_EXCEPTION +) + +marionette_harness_tests_config_options = [ + [['--tests'], { + 'dest': 'test_path', + 'default': None, + 'help': 'Path to test_*.py or directory relative to src root.', + }], + [['--src-dir'], { + 'dest': 'rel_src_dir', + 'default': None, + 'help': 'Path to hg.mo source checkout relative to work dir.', + }], + +] + copy.deepcopy(virtualenv_config_options) + +marionette_harness_tests_config = { + "find_links": [ + "http://pypi.pub.build.mozilla.org/pub", + ], + "pip_index": False, + # relative to workspace + "rel_src_dir": os.path.join("build", "src"), +} + +class MarionetteHarnessTests(VirtualenvMixin, BuildbotMixin, BaseScript): + + def __init__(self, config_options=None, + all_actions=None, default_actions=None, + *args, **kwargs): + config_options = config_options or marionette_harness_tests_config_options + actions = [ + 'clobber', + 'create-virtualenv', + 'run-tests', + ] + super(MarionetteHarnessTests, self).__init__( + config_options=config_options, + all_actions=all_actions or actions, + default_actions=default_actions or actions, + config=marionette_harness_tests_config, + *args, **kwargs) + + @PreScriptAction('create-virtualenv') + def _pre_create_virtualenv(self, action): + dirs = self.query_abs_dirs() + c = self.config + requirements = os.path.join( + dirs['abs_src_dir'], + 'testing', 'config', + 'marionette_harness_test_requirements.txt' + ) + self.register_virtualenv_module( + requirements=[requirements], + two_pass=True + ) + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + c = self.config + abs_dirs = super(MarionetteHarnessTests, self).query_abs_dirs() + dirs = { + 'abs_src_dir': os.path.abspath( + os.path.join(abs_dirs['base_work_dir'], c['rel_src_dir']) + ), + } + + for key in dirs: + if key not in abs_dirs: + abs_dirs[key] = dirs[key] + self.abs_dirs = abs_dirs + + return self.abs_dirs + + def _get_pytest_status(self, code): + """ + Translate pytest exit code to TH status + + Based on https://github.com/pytest-dev/pytest/blob/master/_pytest/main.py#L21-L26 + """ + if code == 0: + return TBPL_SUCCESS + elif code == 1: + return TBPL_WARNING + elif 1 < code < 6: + self.error("pytest returned exit code: %s" % code) + return TBPL_FAILURE + else: + return TBPL_EXCEPTION + + def run_tests(self): + """Run all the tests""" + dirs = self.query_abs_dirs() + test_relpath = self.config.get( + 'test_path', + os.path.join('testing', 'marionette', + 'harness', 'marionette_harness', 'tests', + 'harness_unit') + ) + test_path = os.path.join(dirs['abs_src_dir'], test_relpath) + self.activate_virtualenv() + import pytest + command = ['-p', 'no:terminalreporter', # disable pytest logging + test_path] + logs = {} + for fmt in ['tbpl', 'mach', 'raw']: + logs[fmt] = os.path.join(dirs['abs_log_dir'], + 'mn-harness_{}.log'.format(fmt)) + command.extend(['--log-'+fmt, logs[fmt]]) + self.info('Calling pytest.main with the following arguments: %s' % command) + status = self._get_pytest_status(pytest.main(command)) + self.read_from_file(logs['tbpl']) + for log in logs.values(): + self.copy_to_upload_dir(log, dest='logs/') + self.buildbot_status(status) + + +if __name__ == '__main__': + script = MarionetteHarnessTests() + script.run_and_exit()