Bug 1331899 - wptrunner changes to support v4 manifests, r=Ms2ger draft
authorJames Graham <james@hoppipolla.co.uk>
Tue, 03 Jan 2017 19:05:06 +0000
changeset 462965 75c4bd0ae48b8e703d2cae88e8af83558cbb0ea9
parent 462964 1345536342b74260eff6d92f021374f4c23996d5
child 462966 b63b96f6e6bf01f7bf12519accbb5931618f4a2e
push id41928
push userbmo:james@hoppipolla.co.uk
push dateWed, 18 Jan 2017 10:52:19 +0000
reviewersMs2ger
bugs1331899
milestone53.0a1
Bug 1331899 - wptrunner changes to support v4 manifests, r=Ms2ger The manifest changes came with some trivial API changes, mostly removing unused arguments. MozReview-Commit-ID: G9RZNds4MIE
testing/web-platform/harness/wptrunner/metadata.py
testing/web-platform/harness/wptrunner/testloader.py
testing/web-platform/harness/wptrunner/update/sync.py
--- a/testing/web-platform/harness/wptrunner/metadata.py
+++ b/testing/web-platform/harness/wptrunner/metadata.py
@@ -15,16 +15,17 @@ from mozlog import structuredlog
 
 import expected
 import manifestupdate
 import testloader
 import wptmanifest
 import wpttest
 from vcs import git
 manifest = None  # Module that will be imported relative to test_root
+manifestitem = None
 
 logger = structuredlog.StructuredLogger("web-platform-tests")
 
 
 def load_test_manifests(serve_root, test_paths):
     do_delayed_imports(serve_root)
     manifest_loader = testloader.ManifestLoader(test_paths, False)
     return manifest_loader.load()
@@ -61,18 +62,18 @@ def update_expected(test_paths, serve_ro
         write_changes(metadata_path, expected_map)
 
     results_changed = [item.test_path for item in expected_map.itervalues() if item.modified]
 
     return unexpected_changes(manifests, change_data, results_changed)
 
 
 def do_delayed_imports(serve_root):
-    global manifest
-    from manifest import manifest
+    global manifest, manifestitem
+    from manifest import manifest, item as manifestitem
 
 
 def files_in_repo(repo_root):
     return git("ls-tree", "-r", "--name-only", "HEAD").split("\n")
 
 
 def rev_range(rev_old, rev_new, symmetric=False):
     joiner = ".." if not symmetric else "..."
@@ -107,17 +108,17 @@ def unexpected_changes(manifests, change
         if paths["url_base"] == "/":
             root_manifest = manifest
             break
     else:
         return []
 
     rv = []
 
-    return [fn for fn, tests in root_manifest if fn in files_changed and change_data.get(fn) != "M"]
+    return [fn for _, fn, _ in root_manifest if fn in files_changed and change_data.get(fn) != "M"]
 
 # For each testrun
 # Load all files and scan for the suite_start entry
 # Build a hash of filename: properties
 # For each different set of properties, gather all chunks
 # For each chunk in the set of chunks, go through all tests
 # for each test, make a map of {conditionals: [(platform, new_value)]}
 # Repeat for each platform
@@ -290,19 +291,23 @@ class ExpectedUpdater(object):
         test.set_result(self.run_info, result)
         del self.test_cache[test_id]
 
 
 def create_test_tree(metadata_path, test_manifest, property_order=None,
                      boolean_properties=None):
     expected_map = {}
     id_test_map = {}
-    exclude_types = frozenset(["stub", "helper", "manual"])
-    include_types = set(manifest.item_types) - exclude_types
-    for test_path, tests in test_manifest.itertypes(*include_types):
+    exclude_types = frozenset(["stub", "helper", "manual", "support", "conformancechecker"])
+    all_types = [item.item_type for item in manifestitem.__dict__.itervalues()
+                 if type(item) == type and
+                 issubclass(item, manifestitem.ManifestItem) and
+                 item.item_type is not None]
+    include_types = set(all_types) - exclude_types
+    for _, test_path, tests in test_manifest.itertypes(*include_types):
         expected_data = load_expected(test_manifest, metadata_path, test_path, tests,
                                       property_order=property_order,
                                       boolean_properties=boolean_properties)
         if expected_data is None:
             expected_data = create_expected(test_manifest,
                                             test_path,
                                             tests,
                                             property_order=property_order,
--- a/testing/web-platform/harness/wptrunner/testloader.py
+++ b/testing/web-platform/harness/wptrunner/testloader.py
@@ -41,34 +41,34 @@ class Unchunked(TestChunker):
     def __call__(self, manifest):
         for item in manifest:
             yield item
 
 
 class HashChunker(TestChunker):
     def __call__(self, manifest):
         chunk_index = self.chunk_number - 1
-        for test_path, tests in manifest:
+        for test_type, test_path, tests in manifest:
             h = int(hashlib.md5(test_path).hexdigest(), 16)
             if h % self.total_chunks == chunk_index:
-                yield test_path, tests
+                yield test_type, test_path, tests
 
 
 class DirectoryHashChunker(TestChunker):
     """Like HashChunker except the directory is hashed.
 
     This ensures that all tests in the same directory end up in the same
     chunk.
     """
     def __call__(self, manifest):
         chunk_index = self.chunk_number - 1
-        for test_path, tests in manifest:
+        for test_type, test_path, tests in manifest:
             h = int(hashlib.md5(os.path.dirname(test_path)).hexdigest(), 16)
             if h % self.total_chunks == chunk_index:
-                yield test_path, tests
+                yield test_type, test_path, tests
 
 
 class EqualTimeChunker(TestChunker):
     def _group_by_directory(self, manifest_items):
         """Split the list of manifest items into a ordered dict that groups tests in
         so that anything in the same subdirectory beyond a depth of 3 is in the same
         group. So all tests in a/b/c, a/b/c/d and a/b/c/e will be grouped together
         and separate to tests in a/b/f
@@ -80,28 +80,28 @@ class EqualTimeChunker(TestChunker):
             def __init__(self, path):
                 self.path = path
                 self.time = 0
                 self.tests = []
 
         by_dir = OrderedDict()
         total_time = 0
 
-        for i, (test_path, tests) in enumerate(manifest_items):
+        for i, (test_type, test_path, tests) in enumerate(manifest_items):
             test_dir = tuple(os.path.split(test_path)[0].split(os.path.sep)[:3])
 
             if not test_dir in by_dir:
                 by_dir[test_dir] = PathData(test_dir)
 
             data = by_dir[test_dir]
             time = sum(wpttest.DEFAULT_TIMEOUT if test.timeout !=
                        "long" else wpttest.LONG_TIMEOUT for test in tests)
             data.time += time
             total_time += time
-            data.tests.append((test_path, tests))
+            data.tests.append((test_type, test_path, tests))
 
         return by_dir, total_time
 
     def _maybe_remove(self, chunks, i, direction):
         """Trial removing a chunk from one chunk to an adjacent one.
 
         :param chunks: - the list of all chunks
         :param i: - the chunk index in the list of chunks to try removing from
@@ -347,24 +347,24 @@ class TestFilter(object):
             for item in include:
                 self.manifest.add_include(test_manifests, item)
 
         if exclude:
             for item in exclude:
                 self.manifest.add_exclude(test_manifests, item)
 
     def __call__(self, manifest_iter):
-        for test_path, tests in manifest_iter:
+        for test_type, test_path, tests in manifest_iter:
             include_tests = set()
             for test in tests:
                 if self.manifest.include(test):
                     include_tests.add(test)
 
             if include_tests:
-                yield test_path, include_tests
+                yield test_type, test_path, include_tests
 
 class TagFilter(object):
     def __init__(self, tags):
         self.tags = set(tags)
 
     def __call__(self, test_iter):
         for test in test_iter:
             if test.tags & self.tags:
@@ -401,24 +401,24 @@ class ManifestLoader(object):
             try:
                 with open(manifest_path) as f:
                     json_data = json.load(f)
             except IOError:
                 #If the existing file doesn't exist just create one from scratch
                 pass
 
         if not json_data:
-            manifest_file = manifest.Manifest(None, url_base)
+            manifest_file = manifest.Manifest(url_base)
         else:
             try:
                 manifest_file = manifest.Manifest.from_json(tests_path, json_data)
             except manifest.ManifestVersionMismatch:
-                manifest_file = manifest.Manifest(None, url_base)
+                manifest_file = manifest.Manifest(url_base)
 
-            manifest_update.update(tests_path, url_base, manifest_file)
+            manifest_update.update(tests_path, manifest_file, True)
 
         manifest.write(manifest_file, manifest_path)
 
     def load_manifest(self, tests_path, metadata_path, url_base="/"):
         manifest_path = os.path.join(metadata_path, "MANIFEST.json")
         if (not os.path.exists(manifest_path) or
             self.force_manifest_update):
             self.update_manifest(manifest_path, tests_path, url_base)
@@ -517,24 +517,24 @@ class TestLoader(object):
         for manifest in sorted(self.manifests.keys(), key=lambda x:x.url_base):
             manifest_iter = iterfilter(self.manifest_filters,
                                        manifest.itertypes(*self.test_types))
             manifest_items.extend(manifest_iter)
 
         if self.chunker is not None:
             manifest_items = self.chunker(manifest_items)
 
-        for test_path, tests in manifest_items:
+        for test_type, test_path, tests in manifest_items:
             manifest_file = iter(tests).next().manifest
             metadata_path = self.manifests[manifest_file]["metadata_path"]
             inherit_metadata, test_metadata = self.load_metadata(manifest_file, metadata_path, test_path)
 
             for test in iterfilter(self.meta_filters,
                                    self.iter_wpttest(inherit_metadata, test_metadata, tests)):
-                yield test_path, test.test_type, test
+                yield test_path, test_type, test
 
     def iter_wpttest(self, inherit_metadata, test_metadata, tests):
         for manifest_test in tests:
             yield self.get_test(manifest_test, inherit_metadata, test_metadata)
 
     def _load_tests(self):
         """Read in the tests from the manifest file and add them to a queue"""
         tests = {"enabled":defaultdict(list),
--- a/testing/web-platform/harness/wptrunner/update/sync.py
+++ b/testing/web-platform/harness/wptrunner/update/sync.py
@@ -119,32 +119,30 @@ class GetSyncTargetCommit(Step):
 
         state.sync_tree.checkout(state.sync_commit.sha1, state.local_branch, force=True)
         self.logger.debug("New base commit is %s" % state.sync_commit.sha1)
 
 
 class LoadManifest(Step):
     """Load the test manifest"""
 
-    provides = ["manifest_path", "test_manifest", "old_manifest"]
+    provides = ["manifest_path", "test_manifest"]
 
     def create(self, state):
         from manifest import manifest
         state.manifest_path = os.path.join(state.metadata_path, "MANIFEST.json")
-        # Conservatively always rebuild the manifest when doing a sync
-        state.old_manifest = manifest.load(state.tests_path, state.manifest_path)
-        state.test_manifest = manifest.Manifest(None, "/")
+        state.test_manifest = manifest.Manifest("/")
 
 
 class UpdateManifest(Step):
     """Update the manifest to match the tests in the sync tree checkout"""
 
     def create(self, state):
         from manifest import manifest, update
-        update.update(state.sync["path"], "/", state.test_manifest)
+        update.update(state.sync["path"], state.test_manifest)
         manifest.write(state.test_manifest, state.manifest_path)
 
 
 class CopyWorkTree(Step):
     """Copy the sync tree over to the destination in the local tree"""
 
     def create(self, state):
         copy_wpt_tree(state.sync_tree,