Bug 1048446 - [python-test] Create a mochitest selftest harness, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 31 May 2017 13:52:01 -0400
changeset 363279 83081a324efb68dacb018a4ea1ef8dea91796dba
parent 363278 d98f79ccd63645f5a2e16683b2fa0be73217be04
child 363280 b2d113409e2195b5e70b17c52364acdf72dd575c
push id32001
push userkwierso@gmail.com
push dateFri, 09 Jun 2017 22:48:20 +0000
treeherdermozilla-central@c4e74cfbf7e9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1048446
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1048446 - [python-test] Create a mochitest selftest harness, r=jmaher This will create a mochitest selftest harness based on |mach python-test|. There is also a basic test that checks whether TEST-PASS and TEST-UNEXPECTED-FAIL work. MozReview-Commit-ID: Jqyhbj7nC6z
moz.build
testing/mochitest/tests/python/conftest.py
testing/mochitest/tests/python/files/test_fail.html
testing/mochitest/tests/python/files/test_pass.html
testing/mochitest/tests/python/python.ini
testing/mochitest/tests/python/test_basic_mochitest_plain.py
--- a/moz.build
+++ b/moz.build
@@ -64,16 +64,17 @@ DIRS += [
     'testing/mozbase',
     'third_party/python',
 ]
 
 if not CONFIG['JS_STANDALONE']:
     # These python manifests are included here so they get picked up without an objdir
     PYTHON_UNITTEST_MANIFESTS += [
         'testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini',
+        'testing/mochitest/tests/python/python.ini',
     ]
 
     CONFIGURE_SUBST_FILES += [
         'tools/update-packaging/Makefile',
     ]
     CONFIGURE_DEFINE_FILES += [
         'mozilla-config.h',
     ]
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/conftest.py
@@ -0,0 +1,178 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import shutil
+import sys
+from argparse import Namespace
+from cStringIO import StringIO
+
+import pytest
+import requests
+
+import mozfile
+import mozinstall
+from manifestparser import TestManifest
+from mozbuild.base import MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+HARNESS_ROOT_NOT_FOUND = """
+Could not find test harness root. Either a build or the 'GECKO_INSTALLER_URL'
+environment variable is required.
+""".lstrip()
+
+
+def filter_action(action, lines):
+    return filter(lambda x: x['action'] == action, lines)
+
+
+def _get_harness_root():
+    # Check if there is a local build
+    harness_root = os.path.join(build.topobjdir, '_tests', 'testing', 'mochitest')
+    if os.path.isdir(harness_root):
+        return harness_root
+
+    # Check if it was previously set up by another test
+    harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', 'mochitest')
+    if os.path.isdir(harness_root):
+        return harness_root
+
+    # Check if there is a test package to download
+    if 'GECKO_INSTALLER_URL' in os.environ:
+        base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
+        test_packages = requests.get(base_url + '/target.test_packages.json').json()
+
+        dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
+        for name in test_packages['mochitest']:
+            url = base_url + '/' + name
+            bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
+
+            r = requests.get(url, stream=True)
+            with open(bundle, 'w+b') as fh:
+                for chunk in r.iter_content(chunk_size=1024):
+                    fh.write(chunk)
+
+            mozfile.extract(bundle, dest)
+
+        return os.path.join(dest, 'mochitest')
+
+    # Couldn't find a harness root, let caller do error handling.
+    return None
+
+
+@pytest.fixture(scope='session')
+def setup_harness_root():
+    harness_root = _get_harness_root()
+    if harness_root:
+        sys.path.insert(0, harness_root)
+
+        # Link the test files to the test package so updates are automatically
+        # picked up. Fallback to copy on Windows.
+        test_root = os.path.join(harness_root, 'tests', 'selftests')
+        if not os.path.exists(test_root):
+            files = os.path.join(here, 'files')
+            if hasattr(os, 'symlink'):
+                os.symlink(files, test_root)
+            else:
+                shutil.copytree(files, test_root)
+
+    elif 'GECKO_INSTALLER_URL' in os.environ:
+        # The mochitest tests will run regardless of whether a build exists or not.
+        # In a local environment, they should simply be skipped if setup fails. But
+        # in automation, we'll need to make sure an error is propagated up.
+        pytest.fail(HARNESS_ROOT_NOT_FOUND)
+    else:
+        # Tests will be marked skipped by the calls to pytest.importorskip() below.
+        # We are purposefully not failing here because running |mach python-test|
+        # without a build is a perfectly valid use case.
+        pass
+
+
+@pytest.fixture(scope='session')
+def binary():
+    try:
+        return build.get_binary_path()
+    except:
+        pass
+
+    app = 'firefox'
+    bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app)
+    if os.path.isdir(bindir):
+        try:
+            return mozinstall.get_binary(bindir, app_name=app)
+        except:
+            pass
+
+    if 'GECKO_INSTALLER_URL' in os.environ:
+        bindir = mozinstall.install(
+            os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
+        return mozinstall.get_binary(bindir, app_name='firefox')
+
+
+@pytest.fixture(scope='function')
+def parser(request):
+    parser = pytest.importorskip('mochitest_options')
+
+    app = getattr(request.module, 'APP', 'generic')
+    return parser.MochitestArgumentParser(app=app)
+
+
+@pytest.fixture(scope='function')
+def runtests(setup_harness_root, binary, parser, request):
+    """Creates an easy to use entry point into the mochitest harness.
+
+    :returns: A function with the signature `*tests, **opts`. Each test is a file name
+              (relative to the `files` dir). At least one is required. The opts are
+              used to override the default mochitest options, they are optional.
+    """
+    runtests = pytest.importorskip('runtests')
+
+    mochitest_root = runtests.SCRIPT_DIR
+    test_root = os.path.join(mochitest_root, 'tests', 'selftests')
+
+    buf = StringIO()
+    options = vars(parser.parse_args([]))
+    options.update({
+        'app': binary,
+        'keep_open': False,
+        'log_raw': [buf],
+    })
+
+    if not os.path.isdir(runtests.build_obj.bindir):
+        package_root = os.path.dirname(mochitest_root)
+        options.update({
+            'certPath': os.path.join(package_root, 'certs'),
+            'utilityPath': os.path.join(package_root, 'bin'),
+        })
+        options['extraProfileFiles'].append(os.path.join(package_root, 'bin', 'plugins'))
+
+    options.update(getattr(request.module, 'OPTIONS', {}))
+
+    def normalize(test):
+        return {
+            'name': test,
+            'relpath': test,
+            'path': os.path.join(test_root, test),
+            # add a dummy manifest file because mochitest expects it
+            'manifest': os.path.join(test_root, 'mochitest.ini'),
+        }
+
+    def inner(*tests, **opts):
+        assert len(tests) > 0
+
+        manifest = TestManifest()
+        manifest.tests.extend(map(normalize, tests))
+        options['manifestFile'] = manifest
+        options.update(opts)
+
+        result = runtests.run_test_harness(parser, Namespace(**options))
+        out = json.loads('[' + ','.join(buf.getvalue().splitlines()) + ']')
+        buf.close()
+        return result, out
+    return inner
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/files/test_fail.html
@@ -0,0 +1,24 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1343659
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test Fail</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript">
+    ok(false, "Test is ok");
+  </script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1343659">Mozilla Bug 1343659</a>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/files/test_pass.html
@@ -0,0 +1,24 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1343659
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test Pass</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript">
+    ok(true, "Test is ok");
+  </script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1343659">Mozilla Bug 1343659</a>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/python.ini
@@ -0,0 +1,5 @@
+[DEFAULT]
+subsuite = mochitest
+sequential = true
+
+[test_basic_mochitest_plain.py]
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/test_basic_mochitest_plain.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import sys
+
+import pytest
+
+from conftest import build, filter_action
+
+sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
+from mozharness.base.log import INFO, WARNING
+from mozharness.base.errors import BaseErrorList
+from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING
+from mozharness.mozilla.structuredlog import StructuredOutputParser
+from mozharness.mozilla.testing.errors import HarnessErrorList
+
+
+def get_mozharness_status(lines, status):
+    parser = StructuredOutputParser(
+        config={'log_level': INFO},
+        error_list=BaseErrorList+HarnessErrorList,
+        strict=False,
+        suite_category='mochitest',
+    )
+
+    for line in lines:
+        parser.parse_single_line(json.dumps(line))
+    return parser.evaluate_parser(status)
+
+
+def test_output_pass(runtests):
+    status, lines = runtests('test_pass.html')
+    assert status == 0
+
+    tbpl_status, log_level = get_mozharness_status(lines, status)
+    assert tbpl_status == TBPL_SUCCESS
+    assert log_level == WARNING
+
+    lines = filter_action('test_status', lines)
+    assert len(lines) == 1
+    assert lines[0]['status'] == 'PASS'
+
+
+def test_output_fail(runtests):
+    from runtests import build_obj
+
+    status, lines = runtests('test_fail.html')
+    assert status == 1
+
+    tbpl_status, log_level = get_mozharness_status(lines, status)
+    assert tbpl_status == TBPL_WARNING
+    assert log_level == WARNING
+
+    lines = filter_action('test_status', lines)
+
+    # If we are running with a build_obj, the failed status will be
+    # logged a second time at the end of the run.
+    if build_obj:
+        assert len(lines) == 2
+    else:
+        assert len(lines) == 1
+    assert lines[0]['status'] == 'FAIL'
+
+    if build_obj:
+        assert set(lines[0].keys()) == set(lines[1].keys())
+        assert set(lines[0].values()) == set(lines[1].values())
+
+
+if __name__ == '__main__':
+    sys.exit(pytest.main(['--verbose', __file__]))