testing/mozbase/moztest/tests/test_resolve.py
author Andrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 22 Apr 2019 22:32:34 +0000
changeset 470488 60c512bab3e986b609d7fda44d827c21d90612d2
parent 466458 0f131b481e5e6eeafd88548741672602dc0ec6c6
child 472064 0e39f1415d9b7edb556eb23fe6fc0a7e3afc4e26
permissions -rw-r--r--
Bug 1523303 - Align mozharness suite names with the ones in 'moztest.resolve', r=gbrown This officially makes 'moztest.resolve' the source of truth when it comes to suite names. It aligns that file with the names used in both the desktop_unittest and android_emulator_unittest scripts. Differential Revision: https://phabricator.services.mozilla.com/D27555

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# flake8: noqa: E501

from __future__ import absolute_import, print_function, unicode_literals

import cPickle as pickle
import os
import re
import shutil
import tempfile
import unittest

import mozpack.path as mozpath
import mozunit
from mozbuild.base import MozbuildObject
from mozfile import NamedTemporaryFile

from moztest.resolve import (
    TestMetadata,
    TestResolver,
    TEST_SUITES,
)


ALL_TESTS = {
    "accessible/tests/mochitest/actions/test_anchors.html": [
        {
            "dir_relpath": "accessible/tests/mochitest/actions",
            "expected": "pass",
            "file_relpath": "accessible/tests/mochitest/actions/test_anchors.html",
            "flavor": "a11y",
            "here": "/firefox/accessible/tests/mochitest/actions",
            "manifest": "/firefox/accessible/tests/mochitest/actions/a11y.ini",
            "name": "test_anchors.html",
            "path": "/firefox/accessible/tests/mochitest/actions/test_anchors.html",
            "relpath": "test_anchors.html"
        }
    ],
    "services/common/tests/unit/test_async_chain.js": [
        {
            "dir_relpath": "services/common/tests/unit",
            "file_relpath": "services/common/tests/unit/test_async_chain.js",
            "firefox-appdir": "browser",
            "flavor": "xpcshell",
            "head": "head_global.js head_helpers.js head_http.js",
            "here": "/firefox/services/common/tests/unit",
            "manifest": "/firefox/services/common/tests/unit/xpcshell.ini",
            "name": "test_async_chain.js",
            "path": "/firefox/services/common/tests/unit/test_async_chain.js",
            "relpath": "test_async_chain.js",
        }
    ],
    "services/common/tests/unit/test_async_querySpinningly.js": [
        {
            "dir_relpath": "services/common/tests/unit",
            "file_relpath": "services/common/tests/unit/test_async_querySpinningly.js",
            "firefox-appdir": "browser",
            "flavor": "xpcshell",
            "head": "head_global.js head_helpers.js head_http.js",
            "here": "/firefox/services/common/tests/unit",
            "manifest": "/firefox/services/common/tests/unit/xpcshell.ini",
            "name": "test_async_querySpinningly.js",
            "path": "/firefox/services/common/tests/unit/test_async_querySpinningly.js",
            "relpath": "test_async_querySpinningly.js",
        }
    ],
    "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js": [
        {
            "dir_relpath": "toolkit/mozapps/update/test/unit",
            "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
            "flavor": "xpcshell",
            "generated-files": "head_update.js",
            "head": "head_update.js",
            "here": "/firefox/toolkit/mozapps/update/test/unit",
            "manifest": "/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
            "name": "test_0201_app_launch_apply_update.js",
            "path": "/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
            "reason": "bug 820380",
            "relpath": "test_0201_app_launch_apply_update.js",
            "run-sequentially": "Launches application.",
            "skip-if": "os == 'android'",
        },
        {
            "dir_relpath": "toolkit/mozapps/update/test/unit",
            "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
            "flavor": "xpcshell",
            "generated-files": "head_update.js",
            "head": "head_update.js head2.js",
            "here": "/firefox/toolkit/mozapps/update/test/unit",
            "manifest": "/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
            "name": "test_0201_app_launch_apply_update.js",
            "path": "/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
            "reason": "bug 820380",
            "relpath": "test_0201_app_launch_apply_update.js",
            "run-sequentially": "Launches application.",
            "skip-if": "os == 'android'",
        }
    ],
    "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java": [
        {
            "dir_relpath": "mobile/android/tests/background/junit3/src/common",
            "file_relpath": "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java",
            "flavor": "instrumentation",
            "here": "/firefox/mobile/android/tests/background/junit3",
            "manifest": "/firefox/mobile/android/tests/background/junit3/instrumentation.ini",
            "name": "src/common/TestAndroidLogWriters.java",
            "path": "/firefox/mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java",
            "relpath": "src/common/TestAndroidLogWriters.java",
            "subsuite": "background"
        }
    ],
    "mobile/android/tests/browser/junit3/src/TestDistribution.java": [
        {
            "dir_relpath": "mobile/android/tests/browser/junit3/src",
            "file_relpath": "mobile/android/tests/browser/junit3/src/TestDistribution.java",
            "flavor": "instrumentation",
            "here": "/firefox/mobile/android/tests/browser/junit3",
            "manifest": "/firefox/mobile/android/tests/browser/junit3/instrumentation.ini",
            "name": "src/TestDistribution.java",
            "path": "/firefox/mobile/android/tests/browser/junit3/src/TestDistribution.java",
            "relpath": "src/TestDistribution.java",
            "subsuite": "browser"
        }
    ],
    "image/test/browser/browser_bug666317.js": [
        {
            "dir_relpath": "image/test/browser",
            "file_relpath": "image/test/browser/browser_bug666317.js",
            "flavor": "browser-chrome",
            "here": "/firefox/testing/mochitest/browser/image/test/browser",
            "manifest": "/firefox/image/test/browser/browser.ini",
            "name": "browser_bug666317.js",
            "path": "/firefox/testing/mochitest/browser/image/test/browser/browser_bug666317.js",
            "relpath": "image/test/browser/browser_bug666317.js",
            "skip-if": "e10s # Bug 948194 - Decoded Images seem to not be discarded on memory-pressure notification with e10s enabled",
            "subsuite": ""
        }
   ],
   "devtools/client/markupview/test/browser_markupview_copy_image_data.js": [
        {
            "dir_relpath": "devtools/client/markupview/test",
            "file_relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
            "flavor": "browser-chrome",
            "here": "/firefox/testing/mochitest/browser/devtools/client/markupview/test",
            "manifest": "/firefox/devtools/client/markupview/test/browser.ini",
            "name": "browser_markupview_copy_image_data.js",
            "path": "/firefox/testing/mochitest/browser/devtools/client/markupview/test/browser_markupview_copy_image_data.js",
            "relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
            "subsuite": "devtools",
            "tags": "devtools"
        }
   ]
}

TEST_DEFAULTS = {
    "/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\ndata/**\nxpcshell_updater.ini"}
}

TASK_LABELS = [
    'test-linux64/opt-browser-screenshots-1',
    'test-linux64/opt-browser-screenshots-e10s-1',
    'test-linux64/opt-marionette',
    'test-linux64/opt-mochitest',
    'test-linux64/debug-mochitest-e10s',
    'test-linux64/opt-mochitest-a11y',
    'test-linux64/opt-mochitest-browser',
    'test-linux64/opt-mochitest-browser-chrome',
    'test-linux64/opt-mochitest-browser-chrome-e10s',
    'test-linux64/opt-mochitest-browser-chrome-e10s-11',
    'test-linux64/opt-mochitest-chrome',
    'test-linux64/opt-mochitest-clipboard',
    'test-linux64/opt-mochitest-devtools',
    'test-linux64/opt-mochitest-devtools-chrome',
    'test-linux64/opt-mochitest-gpu',
    'test-linux64/opt-mochitest-gpu-e10s',
    'test-linux64/opt-mochitest-media-e10s-1',
    'test-linux64/opt-mochitest-media-e10s-11',
    'test-linux64/opt-mochitest-plain',
    'test-linux64/opt-mochitest-screenshots-1',
    'test-linux64/opt-reftest',
    'test-linux64/debug-reftest-e10s-1',
    'test-linux64/debug-reftest-e10s-11',
    'test-linux64/opt-robocop',
    'test-linux64/opt-robocop-1',
    'test-linux64/opt-robocop-e10s',
    'test-linux64/opt-robocop-e10s-1',
    'test-linux64/opt-robocop-e10s-11',
    'test-linux64/opt-web-platform-tests-e10s-1',
    'test-linux64/opt-web-platform-tests-reftests-e10s-1',
    'test-linux64/opt-web-platform-tests-reftest-e10s-1',
    'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
    'test-linux64/opt-web-platform-tests-1',
    'test-linux64/opt-web-platform-test-e10s-1',
    'test-linux64/opt-xpcshell',
    'test-linux64/opt-xpcshell-1',
    'test-linux64/opt-xpcshell-2',
]


class Base(unittest.TestCase):
    def setUp(self):
        self._temp_files = []

    def tearDown(self):
        for f in self._temp_files:
            del f

        self._temp_files = []

    def _get_test_metadata(self):
        all_tests = NamedTemporaryFile(mode='wb')
        pickle.dump(ALL_TESTS, all_tests)
        all_tests.flush()
        self._temp_files.append(all_tests)

        test_defaults = NamedTemporaryFile(mode='wb')
        pickle.dump(TEST_DEFAULTS, test_defaults)
        test_defaults.flush()
        self._temp_files.append(test_defaults)

        rv = TestMetadata(all_tests.name, "/firefox/", test_defaults=test_defaults.name)
        rv._wpt_loaded = True  # Don't try to load the wpt manifest
        return rv


class TestTestMetadata(Base):
    def test_load(self):
        t = self._get_test_metadata()
        self.assertEqual(len(t._tests_by_path), 8)

        self.assertEqual(len(list(t.tests_with_flavor('xpcshell'))), 3)
        self.assertEqual(len(list(t.tests_with_flavor('mochitest-plain'))), 0)

    def test_resolve_all(self):
        t = self._get_test_metadata()
        self.assertEqual(len(list(t.resolve_tests())), 9)

    def test_resolve_filter_flavor(self):
        t = self._get_test_metadata()
        self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell'))), 4)

    def test_resolve_by_dir(self):
        t = self._get_test_metadata()
        self.assertEqual(len(list(t.resolve_tests(paths=['services/common']))), 2)

    def test_resolve_under_path(self):
        t = self._get_test_metadata()
        self.assertEqual(len(list(t.resolve_tests(under_path='services'))), 2)

        self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell',
            under_path='services'))), 2)

    def test_resolve_multiple_paths(self):
        t = self._get_test_metadata()
        result = list(t.resolve_tests(paths=['services', 'toolkit']))
        self.assertEqual(len(result), 4)

    def test_resolve_support_files(self):
        expected_support_files = "\ndata/**\nxpcshell_updater.ini"
        t = self._get_test_metadata()
        result = list(t.resolve_tests(paths=['toolkit']))
        self.assertEqual(len(result), 2)

        for test in result:
            self.assertEqual(test['support-files'],
                             expected_support_files)

    def test_resolve_path_prefix(self):
        t = self._get_test_metadata()
        result = list(t.resolve_tests(paths=['image']))
        self.assertEqual(len(result), 1)


class TestTestResolver(Base):
    FAKE_TOPSRCDIR = '/firefox'

    def setUp(self):
        Base.setUp(self)

        self._temp_dirs = []

    def tearDown(self):
        Base.tearDown(self)

        for d in self._temp_dirs:
            shutil.rmtree(d)

    def _get_resolver(self):
        topobjdir = tempfile.mkdtemp()
        self._temp_dirs.append(topobjdir)

        with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
            pickle.dump(ALL_TESTS, fh)
        with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
            pickle.dump(TEST_DEFAULTS, fh)

        o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir)

        # Monkey patch the test resolver to avoid tests failing to find make
        # due to our fake topscrdir.
        TestResolver._run_make = lambda *a, **b: None

        return o._spawn(TestResolver)

    def test_cwd_children_only(self):
        """If cwd is defined, only resolve tests under the specified cwd."""
        r = self._get_resolver()

        # Pretend we're under '/services' and ask for 'common'. This should
        # pick up all tests from '/services/common'
        tests = list(r.resolve_tests(paths=['common'], cwd=os.path.join(r.topsrcdir,
            'services')))

        self.assertEqual(len(tests), 2)

        # Tests should be rewritten to objdir.
        for t in tests:
            self.assertEqual(t['here'], mozpath.join(r.topobjdir,
                '_tests/xpcshell/services/common/tests/unit'))

    def test_various_cwd(self):
        """Test various cwd conditions are all equal."""

        r = self._get_resolver()

        expected = list(r.resolve_tests(paths=['services']))
        actual = list(r.resolve_tests(paths=['services'], cwd='/'))
        self.assertEqual(actual, expected)

        actual = list(r.resolve_tests(paths=['services'], cwd=r.topsrcdir))
        self.assertEqual(actual, expected)

        actual = list(r.resolve_tests(paths=['services'], cwd=r.topobjdir))
        self.assertEqual(actual, expected)

    def test_subsuites(self):
        """Test filtering by subsuite."""

        r = self._get_resolver()

        tests = list(r.resolve_tests(paths=['mobile']))
        self.assertEqual(len(tests), 2)

        tests = list(r.resolve_tests(paths=['mobile'], subsuite='browser'))
        self.assertEqual(len(tests), 1)
        self.assertEqual(tests[0]['name'], 'src/TestDistribution.java')

        tests = list(r.resolve_tests(paths=['mobile'], subsuite='background'))
        self.assertEqual(len(tests), 1)
        self.assertEqual(tests[0]['name'], 'src/common/TestAndroidLogWriters.java')

    def test_wildcard_patterns(self):
        """Test matching paths by wildcard."""

        r = self._get_resolver()

        tests = list(r.resolve_tests(paths=['mobile/**']))
        self.assertEqual(len(tests), 2)
        for t in tests:
            self.assertTrue(t['file_relpath'].startswith('mobile'))

        tests = list(r.resolve_tests(paths=['**/**.js', 'accessible/**']))
        self.assertEqual(len(tests), 7)
        for t in tests:
            path = t['file_relpath']
            self.assertTrue(path.startswith('accessible') or path.endswith('.js'))

    def test_resolve_metadata(self):
        """Test finding metadata from outgoing files."""
        r = self._get_resolver()

        suites, tests = r.resolve_metadata(['bc'])
        assert suites == {'mochitest-browser-chrome'}
        assert tests == []

        suites, tests = r.resolve_metadata(['mochitest-a11y', '/browser', 'xpcshell'])
        assert suites == {'mochitest-a11y', 'xpcshell'}
        assert sorted(t['file_relpath'] for t in tests) == [
            'devtools/client/markupview/test/browser_markupview_copy_image_data.js',
            'image/test/browser/browser_bug666317.js',
            'mobile/android/tests/browser/junit3/src/TestDistribution.java',
        ]

    def test_task_regexes(self):
        """Test the task_regexes defined in TEST_SUITES."""

        test_cases = {
            'mochitest-browser-chrome': [
                'test-linux64/opt-mochitest-browser-chrome',
                'test-linux64/opt-mochitest-browser-chrome-e10s',
            ],
            'mochitest-chrome': [
                'test-linux64/opt-mochitest-chrome',
            ],
            'mochitest-devtools-chrome': [
                'test-linux64/opt-mochitest-devtools-chrome',
            ],
            'mochitest-media': [
                'test-linux64/opt-mochitest-media-e10s-1',
            ],
            'mochitest-plain': [
                'test-linux64/opt-mochitest',
                'test-linux64/debug-mochitest-e10s',
                # this isn't a real task but the regex would match it if it were
                'test-linux64/opt-mochitest-plain',
            ],
            'mochitest-plain-gpu': [
                'test-linux64/opt-mochitest-gpu',
                'test-linux64/opt-mochitest-gpu-e10s',
            ],
            'mochitest-browser-chrome-screenshots': [
                'test-linux64/opt-browser-screenshots-1',
                'test-linux64/opt-browser-screenshots-e10s-1',
            ],
            'reftest': [
                'test-linux64/opt-reftest',
                'test-linux64/debug-reftest-e10s-1',
            ],
            'robocop': [
                'test-linux64/opt-robocop',
                'test-linux64/opt-robocop-1',
                'test-linux64/opt-robocop-e10s',
                'test-linux64/opt-robocop-e10s-1',
            ],
            'web-platform-tests': [
                'test-linux64/opt-web-platform-tests-e10s-1',
                'test-linux64/opt-web-platform-tests-reftests-e10s-1',
                'test-linux64/opt-web-platform-tests-reftest-e10s-1',
                'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
                'test-linux64/opt-web-platform-tests-1',
            ],
            'web-platform-tests-testharness': [
                'test-linux64/opt-web-platform-tests-e10s-1',
                'test-linux64/opt-web-platform-tests-1',
            ],
            'web-platform-tests-reftest': [
                'test-linux64/opt-web-platform-tests-reftests-e10s-1',
            ],
            'web-platform-tests-wdspec': [
                'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
            ],
            'xpcshell': [
                'test-linux64/opt-xpcshell',
                'test-linux64/opt-xpcshell-1',
            ],
        }

        regexes = []

        def match_task(task):
            return any(re.search(pattern, task) for pattern in regexes)

        for suite, expected in sorted(test_cases.items()):
            print(suite)
            regexes = TEST_SUITES[suite]['task_regex']
            assert set(filter(match_task, TASK_LABELS)) == set(expected)


if __name__ == '__main__':
    mozunit.main()