Bug 1048446 - [python-test] Create a mochitest selftest harness, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 31 May 2017 13:52:01 -0400
changeset 592038 83081a324efb68dacb018a4ea1ef8dea91796dba
parent 592037 d98f79ccd63645f5a2e16683b2fa0be73217be04
child 592039 b2d113409e2195b5e70b17c52364acdf72dd575c
push id63265
push userbmo:sfoster@mozilla.com
push dateSat, 10 Jun 2017 01:07:46 +0000
reviewersjmaher
bugs1048446
milestone55.0a1
Bug 1048446 - [python-test] Create a mochitest selftest harness, r=jmaher This will create a mochitest selftest harness based on |mach python-test|. There is also a basic test that checks whether TEST-PASS and TEST-UNEXPECTED-FAIL work. MozReview-Commit-ID: Jqyhbj7nC6z
moz.build
testing/mochitest/tests/python/conftest.py
testing/mochitest/tests/python/files/test_fail.html
testing/mochitest/tests/python/files/test_pass.html
testing/mochitest/tests/python/python.ini
testing/mochitest/tests/python/test_basic_mochitest_plain.py
--- a/moz.build
+++ b/moz.build
@@ -64,16 +64,17 @@ DIRS += [
     'testing/mozbase',
     'third_party/python',
 ]
 
 if not CONFIG['JS_STANDALONE']:
     # These python manifests are included here so they get picked up without an objdir
     PYTHON_UNITTEST_MANIFESTS += [
         'testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini',
+        'testing/mochitest/tests/python/python.ini',
     ]
 
     CONFIGURE_SUBST_FILES += [
         'tools/update-packaging/Makefile',
     ]
     CONFIGURE_DEFINE_FILES += [
         'mozilla-config.h',
     ]
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/conftest.py
@@ -0,0 +1,178 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import shutil
+import sys
+from argparse import Namespace
+from cStringIO import StringIO
+
+import pytest
+import requests
+
+import mozfile
+import mozinstall
+from manifestparser import TestManifest
+from mozbuild.base import MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+HARNESS_ROOT_NOT_FOUND = """
+Could not find test harness root. Either a build or the 'GECKO_INSTALLER_URL'
+environment variable is required.
+""".lstrip()
+
+
+def filter_action(action, lines):
+    return filter(lambda x: x['action'] == action, lines)
+
+
+def _get_harness_root():
+    # Check if there is a local build
+    harness_root = os.path.join(build.topobjdir, '_tests', 'testing', 'mochitest')
+    if os.path.isdir(harness_root):
+        return harness_root
+
+    # Check if it was previously set up by another test
+    harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', 'mochitest')
+    if os.path.isdir(harness_root):
+        return harness_root
+
+    # Check if there is a test package to download
+    if 'GECKO_INSTALLER_URL' in os.environ:
+        base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
+        test_packages = requests.get(base_url + '/target.test_packages.json').json()
+
+        dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
+        for name in test_packages['mochitest']:
+            url = base_url + '/' + name
+            bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
+
+            r = requests.get(url, stream=True)
+            with open(bundle, 'w+b') as fh:
+                for chunk in r.iter_content(chunk_size=1024):
+                    fh.write(chunk)
+
+            mozfile.extract(bundle, dest)
+
+        return os.path.join(dest, 'mochitest')
+
+    # Couldn't find a harness root, let caller do error handling.
+    return None
+
+
+@pytest.fixture(scope='session')
+def setup_harness_root():
+    harness_root = _get_harness_root()
+    if harness_root:
+        sys.path.insert(0, harness_root)
+
+        # Link the test files to the test package so updates are automatically
+        # picked up. Fallback to copy on Windows.
+        test_root = os.path.join(harness_root, 'tests', 'selftests')
+        if not os.path.exists(test_root):
+            files = os.path.join(here, 'files')
+            if hasattr(os, 'symlink'):
+                os.symlink(files, test_root)
+            else:
+                shutil.copytree(files, test_root)
+
+    elif 'GECKO_INSTALLER_URL' in os.environ:
+        # The mochitest tests will run regardless of whether a build exists or not.
+        # In a local environment, they should simply be skipped if setup fails. But
+        # in automation, we'll need to make sure an error is propagated up.
+        pytest.fail(HARNESS_ROOT_NOT_FOUND)
+    else:
+        # Tests will be marked skipped by the calls to pytest.importorskip() below.
+        # We are purposefully not failing here because running |mach python-test|
+        # without a build is a perfectly valid use case.
+        pass
+
+
+@pytest.fixture(scope='session')
+def binary():
+    try:
+        return build.get_binary_path()
+    except:
+        pass
+
+    app = 'firefox'
+    bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app)
+    if os.path.isdir(bindir):
+        try:
+            return mozinstall.get_binary(bindir, app_name=app)
+        except:
+            pass
+
+    if 'GECKO_INSTALLER_URL' in os.environ:
+        bindir = mozinstall.install(
+            os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
+        return mozinstall.get_binary(bindir, app_name='firefox')
+
+
+@pytest.fixture(scope='function')
+def parser(request):
+    parser = pytest.importorskip('mochitest_options')
+
+    app = getattr(request.module, 'APP', 'generic')
+    return parser.MochitestArgumentParser(app=app)
+
+
+@pytest.fixture(scope='function')
+def runtests(setup_harness_root, binary, parser, request):
+    """Creates an easy to use entry point into the mochitest harness.
+
+    :returns: A function with the signature `*tests, **opts`. Each test is a file name
+              (relative to the `files` dir). At least one is required. The opts are
+              used to override the default mochitest options, they are optional.
+    """
+    runtests = pytest.importorskip('runtests')
+
+    mochitest_root = runtests.SCRIPT_DIR
+    test_root = os.path.join(mochitest_root, 'tests', 'selftests')
+
+    buf = StringIO()
+    options = vars(parser.parse_args([]))
+    options.update({
+        'app': binary,
+        'keep_open': False,
+        'log_raw': [buf],
+    })
+
+    if not os.path.isdir(runtests.build_obj.bindir):
+        package_root = os.path.dirname(mochitest_root)
+        options.update({
+            'certPath': os.path.join(package_root, 'certs'),
+            'utilityPath': os.path.join(package_root, 'bin'),
+        })
+        options['extraProfileFiles'].append(os.path.join(package_root, 'bin', 'plugins'))
+
+    options.update(getattr(request.module, 'OPTIONS', {}))
+
+    def normalize(test):
+        return {
+            'name': test,
+            'relpath': test,
+            'path': os.path.join(test_root, test),
+            # add a dummy manifest file because mochitest expects it
+            'manifest': os.path.join(test_root, 'mochitest.ini'),
+        }
+
+    def inner(*tests, **opts):
+        assert len(tests) > 0
+
+        manifest = TestManifest()
+        manifest.tests.extend(map(normalize, tests))
+        options['manifestFile'] = manifest
+        options.update(opts)
+
+        result = runtests.run_test_harness(parser, Namespace(**options))
+        out = json.loads('[' + ','.join(buf.getvalue().splitlines()) + ']')
+        buf.close()
+        return result, out
+    return inner
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/files/test_fail.html
@@ -0,0 +1,24 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1343659
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test Fail</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript">
+    ok(false, "Test is ok");
+  </script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1343659">Mozilla Bug 1343659</a>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/files/test_pass.html
@@ -0,0 +1,24 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1343659
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test Pass</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript">
+    ok(true, "Test is ok");
+  </script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1343659">Mozilla Bug 1343659</a>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/python.ini
@@ -0,0 +1,5 @@
+[DEFAULT]
+subsuite = mochitest
+sequential = true
+
+[test_basic_mochitest_plain.py]
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/test_basic_mochitest_plain.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import sys
+
+import pytest
+
+from conftest import build, filter_action
+
+sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
+from mozharness.base.log import INFO, WARNING
+from mozharness.base.errors import BaseErrorList
+from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING
+from mozharness.mozilla.structuredlog import StructuredOutputParser
+from mozharness.mozilla.testing.errors import HarnessErrorList
+
+
+def get_mozharness_status(lines, status):
+    parser = StructuredOutputParser(
+        config={'log_level': INFO},
+        error_list=BaseErrorList+HarnessErrorList,
+        strict=False,
+        suite_category='mochitest',
+    )
+
+    for line in lines:
+        parser.parse_single_line(json.dumps(line))
+    return parser.evaluate_parser(status)
+
+
+def test_output_pass(runtests):
+    status, lines = runtests('test_pass.html')
+    assert status == 0
+
+    tbpl_status, log_level = get_mozharness_status(lines, status)
+    assert tbpl_status == TBPL_SUCCESS
+    assert log_level == WARNING
+
+    lines = filter_action('test_status', lines)
+    assert len(lines) == 1
+    assert lines[0]['status'] == 'PASS'
+
+
+def test_output_fail(runtests):
+    from runtests import build_obj
+
+    status, lines = runtests('test_fail.html')
+    assert status == 1
+
+    tbpl_status, log_level = get_mozharness_status(lines, status)
+    assert tbpl_status == TBPL_WARNING
+    assert log_level == WARNING
+
+    lines = filter_action('test_status', lines)
+
+    # If we are running with a build_obj, the failed status will be
+    # logged a second time at the end of the run.
+    if build_obj:
+        assert len(lines) == 2
+    else:
+        assert len(lines) == 1
+    assert lines[0]['status'] == 'FAIL'
+
+    if build_obj:
+        assert set(lines[0].keys()) == set(lines[1].keys())
+        assert set(lines[0].values()) == set(lines[1].values())
+
+
+if __name__ == '__main__':
+    sys.exit(pytest.main(['--verbose', __file__]))