Bug 1331899 - wptrunner changes to support v4 manifests, r=Ms2ger
authorJames Graham <james@hoppipolla.co.uk>
Tue, 03 Jan 2017 19:05:06 +0000
changeset 375122 6e93fdded1289192335788d768fd19381521a86f
parent 375121 b16ebc0970fbd699e7012610ff74a518e5d8a328
child 375123 496a7780461a7bcf3ebdd7675d9ba9198b25c531
push id6996
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 20:48:21 +0000
treeherdermozilla-beta@d89512dab048 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1331899 - wptrunner changes to support v4 manifests, r=Ms2ger The manifest changes came with some trivial API changes, mostly removing unused arguments. MozReview-Commit-ID: G9RZNds4MIE
--- a/testing/web-platform/harness/wptrunner/metadata.py
+++ b/testing/web-platform/harness/wptrunner/metadata.py
@@ -15,16 +15,17 @@ from mozlog import structuredlog
 import expected
 import manifestupdate
 import testloader
 import wptmanifest
 import wpttest
 from vcs import git
 manifest = None  # Module that will be imported relative to test_root
+manifestitem = None
 logger = structuredlog.StructuredLogger("web-platform-tests")
 def load_test_manifests(serve_root, test_paths):
     manifest_loader = testloader.ManifestLoader(test_paths, False)
     return manifest_loader.load()
@@ -61,18 +62,18 @@ def update_expected(test_paths, serve_ro
         write_changes(metadata_path, expected_map)
     results_changed = [item.test_path for item in expected_map.itervalues() if item.modified]
     return unexpected_changes(manifests, change_data, results_changed)
 def do_delayed_imports(serve_root):
-    global manifest
-    from manifest import manifest
+    global manifest, manifestitem
+    from manifest import manifest, item as manifestitem
 def files_in_repo(repo_root):
     return git("ls-tree", "-r", "--name-only", "HEAD").split("\n")
 def rev_range(rev_old, rev_new, symmetric=False):
     joiner = ".." if not symmetric else "..."
@@ -107,17 +108,17 @@ def unexpected_changes(manifests, change
         if paths["url_base"] == "/":
             root_manifest = manifest
         return []
     rv = []
-    return [fn for fn, tests in root_manifest if fn in files_changed and change_data.get(fn) != "M"]
+    return [fn for _, fn, _ in root_manifest if fn in files_changed and change_data.get(fn) != "M"]
 # For each testrun
 # Load all files and scan for the suite_start entry
 # Build a hash of filename: properties
 # For each different set of properties, gather all chunks
 # For each chunk in the set of chunks, go through all tests
 # for each test, make a map of {conditionals: [(platform, new_value)]}
 # Repeat for each platform
@@ -290,19 +291,23 @@ class ExpectedUpdater(object):
         test.set_result(self.run_info, result)
         del self.test_cache[test_id]
 def create_test_tree(metadata_path, test_manifest, property_order=None,
     expected_map = {}
     id_test_map = {}
-    exclude_types = frozenset(["stub", "helper", "manual"])
-    include_types = set(manifest.item_types) - exclude_types
-    for test_path, tests in test_manifest.itertypes(*include_types):
+    exclude_types = frozenset(["stub", "helper", "manual", "support", "conformancechecker"])
+    all_types = [item.item_type for item in manifestitem.__dict__.itervalues()
+                 if type(item) == type and
+                 issubclass(item, manifestitem.ManifestItem) and
+                 item.item_type is not None]
+    include_types = set(all_types) - exclude_types
+    for _, test_path, tests in test_manifest.itertypes(*include_types):
         expected_data = load_expected(test_manifest, metadata_path, test_path, tests,
         if expected_data is None:
             expected_data = create_expected(test_manifest,
--- a/testing/web-platform/harness/wptrunner/testloader.py
+++ b/testing/web-platform/harness/wptrunner/testloader.py
@@ -41,34 +41,34 @@ class Unchunked(TestChunker):
     def __call__(self, manifest):
         for item in manifest:
             yield item
 class HashChunker(TestChunker):
     def __call__(self, manifest):
         chunk_index = self.chunk_number - 1
-        for test_path, tests in manifest:
+        for test_type, test_path, tests in manifest:
             h = int(hashlib.md5(test_path).hexdigest(), 16)
             if h % self.total_chunks == chunk_index:
-                yield test_path, tests
+                yield test_type, test_path, tests
 class DirectoryHashChunker(TestChunker):
     """Like HashChunker except the directory is hashed.
     This ensures that all tests in the same directory end up in the same
     def __call__(self, manifest):
         chunk_index = self.chunk_number - 1
-        for test_path, tests in manifest:
+        for test_type, test_path, tests in manifest:
             h = int(hashlib.md5(os.path.dirname(test_path)).hexdigest(), 16)
             if h % self.total_chunks == chunk_index:
-                yield test_path, tests
+                yield test_type, test_path, tests
 class EqualTimeChunker(TestChunker):
     def _group_by_directory(self, manifest_items):
         """Split the list of manifest items into a ordered dict that groups tests in
         so that anything in the same subdirectory beyond a depth of 3 is in the same
         group. So all tests in a/b/c, a/b/c/d and a/b/c/e will be grouped together
         and separate to tests in a/b/f
@@ -80,28 +80,28 @@ class EqualTimeChunker(TestChunker):
             def __init__(self, path):
                 self.path = path
                 self.time = 0
                 self.tests = []
         by_dir = OrderedDict()
         total_time = 0
-        for i, (test_path, tests) in enumerate(manifest_items):
+        for i, (test_type, test_path, tests) in enumerate(manifest_items):
             test_dir = tuple(os.path.split(test_path)[0].split(os.path.sep)[:3])
             if not test_dir in by_dir:
                 by_dir[test_dir] = PathData(test_dir)
             data = by_dir[test_dir]
             time = sum(wpttest.DEFAULT_TIMEOUT if test.timeout !=
                        "long" else wpttest.LONG_TIMEOUT for test in tests)
             data.time += time
             total_time += time
-            data.tests.append((test_path, tests))
+            data.tests.append((test_type, test_path, tests))
         return by_dir, total_time
     def _maybe_remove(self, chunks, i, direction):
         """Trial removing a chunk from one chunk to an adjacent one.
         :param chunks: - the list of all chunks
         :param i: - the chunk index in the list of chunks to try removing from
@@ -347,24 +347,24 @@ class TestFilter(object):
             for item in include:
                 self.manifest.add_include(test_manifests, item)
         if exclude:
             for item in exclude:
                 self.manifest.add_exclude(test_manifests, item)
     def __call__(self, manifest_iter):
-        for test_path, tests in manifest_iter:
+        for test_type, test_path, tests in manifest_iter:
             include_tests = set()
             for test in tests:
                 if self.manifest.include(test):
             if include_tests:
-                yield test_path, include_tests
+                yield test_type, test_path, include_tests
 class TagFilter(object):
     def __init__(self, tags):
         self.tags = set(tags)
     def __call__(self, test_iter):
         for test in test_iter:
             if test.tags & self.tags:
@@ -401,24 +401,24 @@ class ManifestLoader(object):
                 with open(manifest_path) as f:
                     json_data = json.load(f)
             except IOError:
                 #If the existing file doesn't exist just create one from scratch
         if not json_data:
-            manifest_file = manifest.Manifest(None, url_base)
+            manifest_file = manifest.Manifest(url_base)
                 manifest_file = manifest.Manifest.from_json(tests_path, json_data)
             except manifest.ManifestVersionMismatch:
-                manifest_file = manifest.Manifest(None, url_base)
+                manifest_file = manifest.Manifest(url_base)
-            manifest_update.update(tests_path, url_base, manifest_file)
+            manifest_update.update(tests_path, manifest_file, True)
         manifest.write(manifest_file, manifest_path)
     def load_manifest(self, tests_path, metadata_path, url_base="/"):
         manifest_path = os.path.join(metadata_path, "MANIFEST.json")
         if (not os.path.exists(manifest_path) or
             self.update_manifest(manifest_path, tests_path, url_base)
@@ -517,24 +517,24 @@ class TestLoader(object):
         for manifest in sorted(self.manifests.keys(), key=lambda x:x.url_base):
             manifest_iter = iterfilter(self.manifest_filters,
         if self.chunker is not None:
             manifest_items = self.chunker(manifest_items)
-        for test_path, tests in manifest_items:
+        for test_type, test_path, tests in manifest_items:
             manifest_file = iter(tests).next().manifest
             metadata_path = self.manifests[manifest_file]["metadata_path"]
             inherit_metadata, test_metadata = self.load_metadata(manifest_file, metadata_path, test_path)
             for test in iterfilter(self.meta_filters,
                                    self.iter_wpttest(inherit_metadata, test_metadata, tests)):
-                yield test_path, test.test_type, test
+                yield test_path, test_type, test
     def iter_wpttest(self, inherit_metadata, test_metadata, tests):
         for manifest_test in tests:
             yield self.get_test(manifest_test, inherit_metadata, test_metadata)
     def _load_tests(self):
         """Read in the tests from the manifest file and add them to a queue"""
         tests = {"enabled":defaultdict(list),
--- a/testing/web-platform/harness/wptrunner/update/sync.py
+++ b/testing/web-platform/harness/wptrunner/update/sync.py
@@ -119,32 +119,30 @@ class GetSyncTargetCommit(Step):
         state.sync_tree.checkout(state.sync_commit.sha1, state.local_branch, force=True)
         self.logger.debug("New base commit is %s" % state.sync_commit.sha1)
 class LoadManifest(Step):
     """Load the test manifest"""
-    provides = ["manifest_path", "test_manifest", "old_manifest"]
+    provides = ["manifest_path", "test_manifest"]
     def create(self, state):
         from manifest import manifest
         state.manifest_path = os.path.join(state.metadata_path, "MANIFEST.json")
-        # Conservatively always rebuild the manifest when doing a sync
-        state.old_manifest = manifest.load(state.tests_path, state.manifest_path)
-        state.test_manifest = manifest.Manifest(None, "/")
+        state.test_manifest = manifest.Manifest("/")
 class UpdateManifest(Step):
     """Update the manifest to match the tests in the sync tree checkout"""
     def create(self, state):
         from manifest import manifest, update
-        update.update(state.sync["path"], "/", state.test_manifest)
+        update.update(state.sync["path"], state.test_manifest)
         manifest.write(state.test_manifest, state.manifest_path)
 class CopyWorkTree(Step):
     """Copy the sync tree over to the destination in the local tree"""
     def create(self, state):