Add a 'mochitest selftest' python-test task that depends on a build draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 06 Feb 2017 09:39:31 -0500
changeset 496172 4ef47b57d989ae34206ac02d3ad9d618a982c6f4
parent 496171 880a819155fff741854c03f6685b589fcbb9d0b9
child 548562 afd1a17e7d33d6e896f50a93ed4172e13ec0ca98
push id48542
push userahalberstadt@mozilla.com
push dateThu, 09 Mar 2017 21:53:13 +0000
milestone55.0a1
Add a 'mochitest selftest' python-test task that depends on a build MozReview-Commit-ID: Jqyhbj7nC6z
moz.build
taskcluster/ci/source-test/python-tests.yml
testing/mochitest/mochitest_options.py
testing/mochitest/tests/python/conftest.py
testing/mochitest/tests/python/files/mochitest.ini
testing/mochitest/tests/python/files/test_pass.html
testing/mochitest/tests/python/mochicheck.py
testing/mochitest/tests/python/python.ini
testing/mochitest/tests/python/test_mochitest.py
--- a/moz.build
+++ b/moz.build
@@ -63,16 +63,17 @@ DIRS += [
     'taskcluster',
     'testing/mozbase',
 ]
 
 if not CONFIG['JS_STANDALONE']:
     # These python manifests are included here so they get picked up without an objdir
     PYTHON_UNITTEST_MANIFESTS += [
         'testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini',
+        'testing/mochitest/tests/python/python.ini',
     ]
 
     CONFIGURE_SUBST_FILES += [
         'tools/update-packaging/Makefile',
     ]
     CONFIGURE_DEFINE_FILES += [
         'mozilla-config.h',
     ]
--- a/taskcluster/ci/source-test/python-tests.yml
+++ b/taskcluster/ci/source-test/python-tests.yml
@@ -46,16 +46,51 @@ marionette-harness/opt:
         - integration
         - release
     when:
         files-changed:
           - 'testing/marionette/harness/**'
           - 'testing/mozbase/mozlog/mozlog/pytest_mozlog/**'
           - 'python/mach_commands.py'
 
+mochitest/opt:
+    description: testing/mochitest unittests
+    platforms:
+        - linux64/opt
+    treeherder:
+        symbol: py(mch)
+        kind: test
+        tier: 2
+    worker-type:
+        by-platform:
+            linux64.*: aws-provisioner-v1/b2gtest
+    worker:
+        by-platform:
+            linux64.*:
+                implementation: docker-worker
+                docker-image: {in-tree: "desktop1604-test"}
+                max-run-time: 3600
+    run:
+        using: run-task
+        command: >
+            source /home/worker/scripts/xvfb.sh &&
+            start_xvfb '1600x1200x24' 0 &&
+            cd /home/worker/checkouts/gecko &&
+            ./mach python-test --subsuite mochitest
+        requires-build: true
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - 'config/mozunit.py'
+            - 'python/mach_commands.py'
+            - 'testing/mochitest/**'
+            - 'testing/profiles/prefs_general.js'
+
 mozbase/opt:
     description: testing/mozbase unit tests
     platforms:
         - linux64/opt
     treeherder:
         symbol: py(mb)
         kind: test
         tier: 2
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -755,34 +755,29 @@ class MochitestArguments(ArgumentContain
         if options.jscov_dir_prefix:
             options.jscov_dir_prefix = os.path.abspath(options.jscov_dir_prefix)
             if not os.path.isdir(options.jscov_dir_prefix):
                 parser.error(
                     "directory %s does not exist as a destination for coverage "
                     "data." % options.jscov_dir_prefix)
 
         if options.testingModulesDir is None:
+            # Try to guess the testing modules directory.
+            possible = [os.path.join(here, os.path.pardir, 'modules')]
             if build_obj:
-                options.testingModulesDir = os.path.join(
-                    build_obj.topobjdir, '_tests', 'modules')
-            else:
-                # Try to guess the testing modules directory.
-                # This somewhat grotesque hack allows the buildbot machines to find the
-                # modules directory without having to configure the buildbot hosts. This
-                # code should never be executed in local runs because the build system
-                # should always set the flag that populates this variable. If buildbot ever
-                # passes this argument, this code can be deleted.
-                possible = os.path.join(here, os.path.pardir, 'modules')
+                possible.insert(0, os.path.join(build_obj.topobjdir, '_tests', 'modules'))
 
-                if os.path.isdir(possible):
-                    options.testingModulesDir = possible
+            for p in possible:
+                if os.path.isdir(p):
+                    options.testingModulesDir = p
+                    break
 
         if build_obj:
             plugins_dir = os.path.join(build_obj.distdir, 'plugins')
-            if plugins_dir not in options.extraProfileFiles:
+            if os.path.isdir(plugins_dir) and plugins_dir not in options.extraProfileFiles:
                 options.extraProfileFiles.append(plugins_dir)
 
         # Even if buildbot is updated, we still want this, as the path we pass in
         # to the app must be absolute and have proper slashes.
         if options.testingModulesDir is not None:
             options.testingModulesDir = os.path.normpath(
                 options.testingModulesDir)
 
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/conftest.py
@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+from argparse import Namespace
+
+import pytest
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+@pytest.fixture(scope='function')
+def parser(request):
+    parser = pytest.importorskip('mochitest_options')
+
+    app = getattr(request.module, 'APP', 'generic')
+    return parser.MochitestArgumentParser(app=app)
+
+
+@pytest.fixture(scope='function')
+def runtests(parser, request):
+    runtests = pytest.importorskip('runtests')
+    mochitest_root = runtests.SCRIPT_DIR
+
+    options = vars(parser.parse_args([]))
+    options.update({
+        'app': os.environ.get('GECKO_BINARY_PATH'),
+        'keep_open': False,
+        'manifestFile': os.path.join(mochitest_root, 'tests', 'selftests', 'mochitest.ini'),
+    })
+
+    if runtests.build_obj:
+        package_root = os.path.dirname(mochitest_root)
+        options.update({
+            'certPath': os.path.join(package_root, 'certs'),
+            'utilityPath': os.path.join(package_root, 'bin'),
+        })
+        options['extraProfileFiles'].append(os.path.join(package_root, 'bin', 'plugins'))
+
+    options.update(getattr(request.module, 'OPTIONS', {}))
+
+    def inner(opts=None):
+        opts = opts or {}
+        options.update(opts)
+        return runtests.run_test_harness(parser, Namespace(**options))
+
+    return inner
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/files/mochitest.ini
@@ -0,0 +1,1 @@
+[test_pass.html]
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/files/test_pass.html
@@ -0,0 +1,24 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1343659
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test for Bug 1343659</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript">
+    ok(true, "Test is ok");
+  </script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1343659">Mozilla Bug 1343659</a>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/mochicheck.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import sys
+
+import pytest
+import requests
+
+import mozfile
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+try:
+    from mozbuild.base import MozbuildObject
+    build = MozbuildObject.from_environment(cwd=here)
+except ImportError:
+    build = None
+
+
+HARNESS_ROOT_NOT_FOUND = """
+Could not find test harness root. Either a build or the 'GECKO_INSTALLER_URL'
+environment variable is required.
+""".lstrip()
+
+
+def get_harness_root():
+    if build:
+        harness_root = os.path.join(build.topobjdir, '_tests', 'testing', 'mochitest')
+        if os.path.isdir(harness_root):
+            return harness_root
+
+    # Try to find a test package to download
+    if 'GECKO_INSTALLER_URL' in os.environ:
+        base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
+        test_packages = requests.get(base_url + '/target.test_packages.json').json()
+
+        dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
+        for name in test_packages['mochitest']:
+            url = base_url + '/' + name
+            bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
+
+            r = requests.get(url, stream=True)
+            with open(bundle, 'w+b') as fh:
+                for chunk in r.iter_content(chunk_size=1024):
+                    fh.write(chunk)
+
+            mozfile.extract(bundle, dest)
+
+        return os.path.join(dest, 'mochitest')
+
+
+def runtests():
+    harness_root = get_harness_root()
+    print(harness_root)
+    if harness_root:
+        sys.path.insert(0, harness_root)
+
+        selftests = os.path.join(harness_root, 'tests', 'selftests')
+        if not os.path.exists(selftests):
+            print(selftests)
+            print(os.path.join(here, 'files'))
+            os.symlink(os.path.join(here, 'files'), selftests)
+    elif os.environ.get('MOCHICHECK_ERROR_ON_SETUP'):
+        # The mochitest tests will run regardless of whether a build exists or not.
+        # In a local environment, the should simply be skipped if setup fails. But
+        # automation, we'll need to make sure an error is propagated up.
+        print(HARNESS_ROOT_NOT_FOUND)
+        return 1
+
+    # Without harness_root or ERROR_ON_SETUP, tests will be marked as skipped
+    return pytest.main(['--verbose', here])
+
+
+if __name__ == '__main__':
+    sys.exit(runtests())
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/python.ini
@@ -0,0 +1,9 @@
+[DEFAULT]
+subsuite = mochitest
+
+# Group these tests under a single 'runner' because:
+#   A) These tests can't run in parallel with one another
+#   B) They require common setup/teardown across them
+# Once the current python-test runner can support those
+# edge cases, the tests should be listed out here.
+[mochicheck.py]
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/test_mochitest.py
@@ -0,0 +1,18 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import pytest
+
+
+def test_mochitest(runtests):
+    assert 'GECKO_BINARY_PATH' in os.environ
+    assert os.path.isfile(os.environ['GECKO_BINARY_PATH'])
+    assert runtests() == 0
+    assert False
+
+
+if __name__ == '__main__':
+    pytest.main(['--verbose', __file__])