Bug 1646813 - write tests for chunking.get_runtimes and chunking.chunk_manifests for web-platform-tests r=ahal
authorEdwin Takahashi <egao@mozilla.com>
Mon, 22 Jun 2020 22:00:02 +0000
changeset 536967 9720c404522dc72dda24003efd188a6972f8b4db
parent 536966 cf5883d7b082d81d25e4cc12db3fe89d7b22d9c0
child 536968 504febbebd4c52690e86870928a6ba068f74c456
push id119734
push useregao@mozilla.com
push dateTue, 23 Jun 2020 18:06:35 +0000
treeherderautoland@9720c404522d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1646813
milestone79.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1646813 - write tests for chunking.get_runtimes and chunking.chunk_manifests for web-platform-tests r=ahal Changes: - change default return value for `get_runtimes` method to an empty dictionary if data is not found. - add in fixtures and tests for `get_runtimes` and `chunk_manifests` method for web-platform-tests and subsuites. Differential Revision: https://phabricator.services.mozilla.com/D80250
taskcluster/taskgraph/test/python.ini
taskcluster/taskgraph/test/test_util_chunking.py
taskcluster/taskgraph/util/chunking.py
--- a/taskcluster/taskgraph/test/python.ini
+++ b/taskcluster/taskgraph/test/python.ini
@@ -15,16 +15,17 @@ subsuite = taskgraph
 [test_target_tasks.py]
 [test_taskgraph.py]
 [test_taskcluster_yml.py]
 [test_transforms_base.py]
 [test_transforms_job.py]
 [test_try_option_syntax.py]
 [test_util_attributes.py]
 [test_util_bugbug.py]
+[test_util_chunking.py]
 [test_util_docker.py]
 [test_util_parameterization.py]
 [test_util_python_path.py]
 [test_util_runnable_jobs.py]
 [test_util_schema.py]
 [test_util_taskcluster.py]
 [test_util_templates.py]
 [test_util_time.py]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_chunking.py
@@ -0,0 +1,127 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from itertools import combinations
+from six.moves import range
+
+import pytest
+
+from mozunit import main
+from taskgraph.util import chunking
+
+
+@pytest.fixture(scope='module')
+def mock_manifest_runtimes():
+    """Deterministically produce a list of simulated manifest runtimes.
+
+    Args:
+        manifests (list): list of manifests against which simulated manifest
+            runtimes would be paired up to.
+
+    Returns:
+        dict of manifest data paired with a float value representing runtime.
+    """
+    def inner(manifests):
+        manifests = sorted(manifests)
+        # Generate deterministic runtime data.
+        runtimes = [(i/10) ** (i/10) for i in range(len(manifests))]
+        return dict(zip(manifests, runtimes))
+    return inner
+
+
+@pytest.fixture(scope='module')
+def unchunked_manifests():
+    """Produce a list of unchunked manifests to be consumed by test method.
+
+    Args:
+        length (int, optional): number of path elements to keep.
+        cutoff (int, optional): number of generated test paths to remove
+            from the test set if user wants to limit the number of paths.
+
+    Returns:
+        list: list of test paths.
+    """
+    data = ['blueberry', 'nashi', 'peach', 'watermelon']
+
+    def inner(suite, length=2, cutoff=0):
+        if 'web-platform' in suite:
+            suffix = ''
+            prefix = '/'
+        elif 'reftest' in suite:
+            suffix = '.list'
+            prefix = ''
+        else:
+            suffix = '.ini'
+            prefix = ''
+        return [prefix + '/'.join(p) + suffix for p in combinations(data, length)][cutoff:]
+    return inner
+
+
+@pytest.mark.parametrize('platform', ['unix', 'windows', 'android'])
+@pytest.mark.parametrize('suite', ['crashtest', 'reftest', 'web-platform-tests', 'xpcshell'])
+def test_get_runtimes(platform, suite):
+    """Tests that runtime information is returned for known good configurations.
+    """
+    assert chunking.get_runtimes(platform, suite)
+
+
+@pytest.mark.parametrize('test_cases', [
+        ('nonexistent_platform', 'nonexistent_suite', KeyError),
+        ('unix', 'nonexistent_suite', KeyError),
+        ('unix', '', TypeError),
+        ('', '', TypeError),
+        ('', 'nonexistent_suite', TypeError)
+    ])
+def test_get_runtimes_invalid(test_cases):
+    """Ensure get_runtimes() method raises an exception if improper request is made.
+    """
+    platform = test_cases[0]
+    suite = test_cases[1]
+    expected = test_cases[2]
+
+    try:
+        chunking.get_runtimes(platform, suite)
+    except Exception as e:
+        assert isinstance(e, expected)
+
+
+@pytest.mark.parametrize('suite', ['web-platform-tests', 'web-platform-tests-reftests'])
+@pytest.mark.parametrize('chunks', [1, 3, 6, 20])
+def test_chunk_manifests_wpt(mock_manifest_runtimes, unchunked_manifests, suite, chunks):
+    """Tests web-platform-tests and its subsuites chunking process.
+    """
+    # Setup.
+    manifests = unchunked_manifests(suite)
+
+    # Generate the expected results, by generating list of indices that each
+    # manifest should go into and then appending each item to that index.
+    # This method is intentionally different from the way chunking.py performs
+    # chunking for cross-checking.
+    expected = [[] for _ in range(chunks)]
+    indexed = zip(manifests, list(range(0, chunks)) * len(manifests))
+    for i in indexed:
+        expected[i[1]].append(i[0])
+
+    # Call the method under test on unchunked manifests.
+    chunked_manifests = chunking.chunk_manifests(suite, 'unix', chunks, manifests)
+
+    # Assertions and end test.
+    assert chunked_manifests
+    if chunks > len(manifests):
+        # If chunk count exceeds number of manifests, not all chunks will have
+        # manifests.
+        with pytest.raises(AssertionError):
+            assert all(chunked_manifests)
+    else:
+        assert all(chunked_manifests)
+        minimum = min([len(c) for c in chunked_manifests])
+        maximum = max([len(c) for c in chunked_manifests])
+        assert maximum - minimum <= 1
+        assert expected == chunked_manifests
+
+
+if __name__ == '__main__':
+    main()
--- a/taskcluster/taskgraph/util/chunking.py
+++ b/taskcluster/taskgraph/util/chunking.py
@@ -79,29 +79,32 @@ def guess_mozinfo_from_task(task):
     else:
         info['toolkit'] = 'gtk'
 
     return info
 
 
 @memoize
 def get_runtimes(platform, suite_name):
-    if not suite_name:
-        raise TypeError('suite_name should be a value.')
+    if not suite_name or not platform:
+        raise TypeError('suite_name and platform cannot be empty.')
 
     base = os.path.join(GECKO, 'testing', 'runtimes', 'manifest-runtimes-{}.json')
     for key in ('android', 'windows'):
         if key in platform:
             path = base.format(key)
             break
     else:
         path = base.format('unix')
 
+    if not os.path.exists(path):
+        raise IOError('manifest runtime file at {} not found.'.format(path))
+
     with open(path, 'r') as fh:
-        return json.load(fh).get(suite_name)
+        return json.load(fh)[suite_name]
 
 
 def chunk_manifests(suite, platform, chunks, manifests):
     """Run the chunking algorithm.
 
     Args:
         platform (str): Platform used to find runtime info.
         chunks (int): Number of chunks to split manifests into.