Bug 1583353 - [moztest] Use a fixture to generate the test data in test_resolve.py, r=egao
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 01 Nov 2019 20:13:35 +0000
changeset 500226 c7b6e1aef3a054068e1f4bf24a7cb97181ce7abc
parent 500225 f6d096dd754d0df5490193da0414c747dab13aec
child 500227 275c78492eea0289558fb3dbc7e6bd05432760c9
push id114164
push useraiakab@mozilla.com
push dateTue, 05 Nov 2019 10:06:15 +0000
treeherdermozilla-inbound@4d585c7edc76 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersegao
bugs1583353
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1583353 - [moztest] Use a fixture to generate the test data in test_resolve.py, r=egao I'm making this refactor now because a future commit is going to completely re-write the test data (so that it matches the data that a future TestManifestLoader is going to use). With this method, updating the data will be a lot less tedious. Differential Revision: https://phabricator.services.mozilla.com/D51278
testing/mozbase/moztest/tests/test_resolve.py
--- a/testing/mozbase/moztest/tests/test_resolve.py
+++ b/testing/mozbase/moztest/tests/test_resolve.py
@@ -1,215 +1,153 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 # flake8: noqa: E501
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import cPickle as pickle
+import json
 import os
 import re
 import shutil
 import tempfile
+from collections import defaultdict
 
 import pytest
 import mozpack.path as mozpath
 import mozunit
 from mozbuild.base import MozbuildObject
 from mozfile import NamedTemporaryFile
 
 from moztest.resolve import (
     TestResolver,
     TEST_SUITES,
 )
 
 
-ALL_TESTS = {
-    "accessible/tests/mochitest/actions/test_anchors.html": [
-        {
-            "dir_relpath": "accessible/tests/mochitest/actions",
-            "expected": "pass",
-            "file_relpath": "accessible/tests/mochitest/actions/test_anchors.html",
-            "flavor": "a11y",
-            "here": "/firefox/accessible/tests/mochitest/actions",
-            "manifest": "/firefox/accessible/tests/mochitest/actions/a11y.ini",
-            "name": "test_anchors.html",
-            "path": "/firefox/accessible/tests/mochitest/actions/test_anchors.html",
-            "relpath": "test_anchors.html"
-        }
-    ],
-    "services/common/tests/unit/test_async_chain.js": [
-        {
-            "dir_relpath": "services/common/tests/unit",
-            "file_relpath": "services/common/tests/unit/test_async_chain.js",
+@pytest.fixture(scope='module')
+def create_tests():
+    sourcedir = '/firefox'
+
+    def inner(*paths, **defaults):
+        tests = defaultdict(list)
+        for path in paths:
+            if isinstance(path, tuple):
+                path, kwargs = path
+            else:
+                kwargs = {}
+
+            path = mozpath.normpath(path)
+            manifest_name = kwargs.get('flavor', defaults.get('flavor', 'manifest'))
+            manifest = kwargs.pop('manifest', defaults.pop('manifest',
+                                  mozpath.join(mozpath.dirname(path), manifest_name + '.ini')))
+
+            manifest_abspath = mozpath.join(sourcedir, manifest)
+            relpath = mozpath.relpath(path, mozpath.dirname(manifest))
+            test = {
+                'name': relpath,
+                'path': mozpath.join(sourcedir, path),
+                'relpath': relpath,
+                'file_relpath': path,
+                'dir_relpath': mozpath.dirname(path),
+                'here': mozpath.dirname(manifest_abspath),
+                'manifest': manifest_abspath,
+                'manifest_relpath': manifest,
+            }
+            test.update(**defaults)
+            test.update(**kwargs)
+            tests[path].append(test)
+
+        # dump tests to stdout for easier debugging on failure
+        print("The 'create_tests' fixture returned:")
+        print(json.dumps(dict(tests), indent=2, sort_keys=True))
+        return tests
+
+    return inner
+
+
+@pytest.fixture(scope='module')
+def all_tests(create_tests):
+    return create_tests(*[
+        ("accessible/tests/mochitest/actions/test_anchors.html", {
+             "expected": "pass",
+             "flavor": "a11y",
+         }),
+        ("services/common/tests/unit/test_async_chain.js", {
             "firefox-appdir": "browser",
             "flavor": "xpcshell",
             "head": "head_global.js head_helpers.js head_http.js",
-            "here": "/firefox/services/common/tests/unit",
-            "manifest": "/firefox/services/common/tests/unit/xpcshell.ini",
-            "name": "test_async_chain.js",
-            "path": "/firefox/services/common/tests/unit/test_async_chain.js",
-            "relpath": "test_async_chain.js",
-        }
-    ],
-    "services/common/tests/unit/test_async_querySpinningly.js": [
-        {
-            "dir_relpath": "services/common/tests/unit",
-            "file_relpath": "services/common/tests/unit/test_async_querySpinningly.js",
+         }),
+        ("services/common/tests/unit/test_async_querySpinningly.js", {
             "firefox-appdir": "browser",
             "flavor": "xpcshell",
             "head": "head_global.js head_helpers.js head_http.js",
-            "here": "/firefox/services/common/tests/unit",
-            "manifest": "/firefox/services/common/tests/unit/xpcshell.ini",
-            "name": "test_async_querySpinningly.js",
-            "path": "/firefox/services/common/tests/unit/test_async_querySpinningly.js",
-            "relpath": "test_async_querySpinningly.js",
-        }
-    ],
-    "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js": [
-        {
-            "dir_relpath": "toolkit/mozapps/update/test/unit",
-            "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
+         }),
+        ("toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", {
             "flavor": "xpcshell",
             "generated-files": "head_update.js",
             "head": "head_update.js",
-            "here": "/firefox/toolkit/mozapps/update/test/unit",
-            "manifest": "/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
-            "name": "test_0201_app_launch_apply_update.js",
-            "path": "/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
+            "manifest": "toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
             "reason": "bug 820380",
-            "relpath": "test_0201_app_launch_apply_update.js",
             "run-sequentially": "Launches application.",
             "skip-if": "os == 'android'",
-        },
-        {
-            "dir_relpath": "toolkit/mozapps/update/test/unit",
-            "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
+         }),
+        ("toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", {
             "flavor": "xpcshell",
             "generated-files": "head_update.js",
             "head": "head_update.js head2.js",
-            "here": "/firefox/toolkit/mozapps/update/test/unit",
-            "manifest": "/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
-            "name": "test_0201_app_launch_apply_update.js",
-            "path": "/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
+            "manifest": "toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
             "reason": "bug 820380",
-            "relpath": "test_0201_app_launch_apply_update.js",
             "run-sequentially": "Launches application.",
             "skip-if": "os == 'android'",
-        }
-    ],
-    "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java": [
-        {
-            "dir_relpath": "mobile/android/tests/background/junit3/src/common",
-            "file_relpath": "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java",
-            "flavor": "instrumentation",
-            "here": "/firefox/mobile/android/tests/background/junit3",
-            "manifest": "/firefox/mobile/android/tests/background/junit3/instrumentation.ini",
-            "name": "src/common/TestAndroidLogWriters.java",
-            "path": "/firefox/mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java",
-            "relpath": "src/common/TestAndroidLogWriters.java",
-            "subsuite": "background"
-        }
-    ],
-    "mobile/android/tests/browser/junit3/src/TestDistribution.java": [
-        {
-            "dir_relpath": "mobile/android/tests/browser/junit3/src",
-            "file_relpath": "mobile/android/tests/browser/junit3/src/TestDistribution.java",
+         }),
+        ("mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java", {
             "flavor": "instrumentation",
-            "here": "/firefox/mobile/android/tests/browser/junit3",
-            "manifest": "/firefox/mobile/android/tests/browser/junit3/instrumentation.ini",
-            "name": "src/TestDistribution.java",
-            "path": "/firefox/mobile/android/tests/browser/junit3/src/TestDistribution.java",
-            "relpath": "src/TestDistribution.java",
-            "subsuite": "browser"
-        }
-    ],
-    "image/test/browser/browser_bug666317.js": [
-        {
-            "dir_relpath": "image/test/browser",
-            "file_relpath": "image/test/browser/browser_bug666317.js",
-            "flavor": "browser-chrome",
-            "here": "/firefox/testing/mochitest/browser/image/test/browser",
-            "manifest": "/firefox/image/test/browser/browser.ini",
-            "name": "browser_bug666317.js",
-            "path": "/firefox/testing/mochitest/browser/image/test/browser/browser_bug666317.js",
-            "relpath": "image/test/browser/browser_bug666317.js",
-            "skip-if": "e10s # Bug 948194 - Decoded Images seem to not be discarded on memory-pressure notification with e10s enabled",
-            "subsuite": ""
-        }
-   ],
-   "devtools/client/markupview/test/browser_markupview_copy_image_data.js": [
-        {
-            "dir_relpath": "devtools/client/markupview/test",
-            "file_relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
+            "manifest": "mobile/android/tests/background/junit3/instrumentation.ini",
+            "subsuite": "background",
+         }),
+        ("mobile/android/tests/browser/junit3/src/TestDistribution.java", {
+            "flavor": "instrumentation",
+            "manifest": "mobile/android/tests/browser/junit3/instrumentation.ini",
+            "subsuite": "browser",
+         }),
+        ("image/test/browser/browser_bug666317.js", {
             "flavor": "browser-chrome",
-            "here": "/firefox/testing/mochitest/browser/devtools/client/markupview/test",
-            "manifest": "/firefox/devtools/client/markupview/test/browser.ini",
-            "name": "browser_markupview_copy_image_data.js",
-            "path": "/firefox/testing/mochitest/browser/devtools/client/markupview/test/browser_markupview_copy_image_data.js",
-            "relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
+            "manifest": "image/test/browser/browser.ini",
+            "skip-if": "e10s # Bug 948194 - Decoded Images seem to not be discarded on memory-pressure notification with e10s enabled",
+            "subsuite": "",
+         }),
+        ("devtools/client/markupview/test/browser_markupview_copy_image_data.js", {
+            "flavor": "browser-chrome",
+            "manifest": "devtools/client/markupview/test/browser.ini",
             "subsuite": "devtools",
-            "tags": "devtools"
-        }
-   ]
-}
-
-TEST_DEFAULTS = {
-    "/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\ndata/**\nxpcshell_updater.ini"}
-}
+            "tags": "devtools",
+         }),
+    ])
 
-TASK_LABELS = [
-    'test-linux64/opt-browser-screenshots-1',
-    'test-linux64/opt-browser-screenshots-e10s-1',
-    'test-linux64/opt-marionette',
-    'test-linux64/opt-mochitest',
-    'test-linux64/debug-mochitest-e10s',
-    'test-linux64/opt-mochitest-a11y',
-    'test-linux64/opt-mochitest-browser',
-    'test-linux64/opt-mochitest-browser-chrome',
-    'test-linux64/opt-mochitest-browser-chrome-e10s',
-    'test-linux64/opt-mochitest-browser-chrome-e10s-11',
-    'test-linux64/opt-mochitest-chrome',
-    'test-linux64/opt-mochitest-devtools',
-    'test-linux64/opt-mochitest-devtools-chrome',
-    'test-linux64/opt-mochitest-gpu',
-    'test-linux64/opt-mochitest-gpu-e10s',
-    'test-linux64/opt-mochitest-media-e10s-1',
-    'test-linux64/opt-mochitest-media-e10s-11',
-    'test-linux64/opt-mochitest-plain',
-    'test-linux64/opt-mochitest-screenshots-1',
-    'test-linux64/opt-reftest',
-    'test-linux64/debug-reftest-e10s-1',
-    'test-linux64/debug-reftest-e10s-11',
-    'test-linux64/opt-robocop',
-    'test-linux64/opt-robocop-1',
-    'test-linux64/opt-robocop-e10s',
-    'test-linux64/opt-robocop-e10s-1',
-    'test-linux64/opt-robocop-e10s-11',
-    'test-linux64/opt-web-platform-tests-e10s-1',
-    'test-linux64/opt-web-platform-tests-reftests-e10s-1',
-    'test-linux64/opt-web-platform-tests-reftest-e10s-1',
-    'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
-    'test-linux64/opt-web-platform-tests-1',
-    'test-linux64/opt-web-platform-test-e10s-1',
-    'test-linux64/opt-xpcshell',
-    'test-linux64/opt-xpcshell-1',
-    'test-linux64/opt-xpcshell-2',
-]
+
+@pytest.fixture(scope='module')
+def defaults():
+    return {
+        "/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {
+            "support-files": "\ndata/**\nxpcshell_updater.ini"
+        }
+    }
 
 
 @pytest.fixture
-def resolver(tmpdir):
+def resolver(tmpdir, all_tests, defaults):
     topobjdir = tmpdir.mkdir("objdir").strpath
 
     with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
-        pickle.dump(ALL_TESTS, fh)
+        pickle.dump(all_tests, fh)
     with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
-        pickle.dump(TEST_DEFAULTS, fh)
+        pickle.dump(defaults, fh)
 
     o = MozbuildObject('/firefox', None, None, topobjdir=topobjdir)
 
     # Monkey patch the test resolver to avoid tests failing to find make
     # due to our fake topscrdir.
     TestResolver._run_make = lambda *a, **b: None
     resolver = o._spawn(TestResolver)
     resolver._puppeteer_loaded = True
@@ -326,16 +264,54 @@ def test_resolve_metadata(resolver):
     assert sorted(t['file_relpath'] for t in tests) == [
         'devtools/client/markupview/test/browser_markupview_copy_image_data.js',
         'image/test/browser/browser_bug666317.js',
         'mobile/android/tests/browser/junit3/src/TestDistribution.java',
     ]
 
 def test_task_regexes():
     """Test the task_regexes defined in TEST_SUITES."""
+    task_labels = [
+        'test-linux64/opt-browser-screenshots-1',
+        'test-linux64/opt-browser-screenshots-e10s-1',
+        'test-linux64/opt-marionette',
+        'test-linux64/opt-mochitest',
+        'test-linux64/debug-mochitest-e10s',
+        'test-linux64/opt-mochitest-a11y',
+        'test-linux64/opt-mochitest-browser',
+        'test-linux64/opt-mochitest-browser-chrome',
+        'test-linux64/opt-mochitest-browser-chrome-e10s',
+        'test-linux64/opt-mochitest-browser-chrome-e10s-11',
+        'test-linux64/opt-mochitest-chrome',
+        'test-linux64/opt-mochitest-devtools',
+        'test-linux64/opt-mochitest-devtools-chrome',
+        'test-linux64/opt-mochitest-gpu',
+        'test-linux64/opt-mochitest-gpu-e10s',
+        'test-linux64/opt-mochitest-media-e10s-1',
+        'test-linux64/opt-mochitest-media-e10s-11',
+        'test-linux64/opt-mochitest-plain',
+        'test-linux64/opt-mochitest-screenshots-1',
+        'test-linux64/opt-reftest',
+        'test-linux64/debug-reftest-e10s-1',
+        'test-linux64/debug-reftest-e10s-11',
+        'test-linux64/opt-robocop',
+        'test-linux64/opt-robocop-1',
+        'test-linux64/opt-robocop-e10s',
+        'test-linux64/opt-robocop-e10s-1',
+        'test-linux64/opt-robocop-e10s-11',
+        'test-linux64/opt-web-platform-tests-e10s-1',
+        'test-linux64/opt-web-platform-tests-reftests-e10s-1',
+        'test-linux64/opt-web-platform-tests-reftest-e10s-1',
+        'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
+        'test-linux64/opt-web-platform-tests-1',
+        'test-linux64/opt-web-platform-test-e10s-1',
+        'test-linux64/opt-xpcshell',
+        'test-linux64/opt-xpcshell-1',
+        'test-linux64/opt-xpcshell-2',
+    ]
 
     test_cases = {
         'mochitest-browser-chrome': [
             'test-linux64/opt-mochitest-browser-chrome',
             'test-linux64/opt-mochitest-browser-chrome-e10s',
         ],
         'mochitest-chrome': [
             'test-linux64/opt-mochitest-chrome',
@@ -396,13 +372,13 @@ def test_task_regexes():
     regexes = []
 
     def match_task(task):
         return any(re.search(pattern, task) for pattern in regexes)
 
     for suite, expected in sorted(test_cases.items()):
         print(suite)
         regexes = TEST_SUITES[suite]['task_regex']
-        assert set(filter(match_task, TASK_LABELS)) == set(expected)
+        assert set(filter(match_task, task_labels)) == set(expected)
 
 
 if __name__ == '__main__':
     mozunit.main()