Bug 1536804 - Factor out a base class for the wpt metadata compilers, r=ato
authorJames Graham <james@hoppipolla.co.uk>
Thu, 11 Apr 2019 16:40:15 +0000
changeset 469026 7f7a2ccd90510ef9d36b0408627fd6ee39f3add5
parent 469025 18f136a887319c07041e8f9f16d63aaac512a234
child 469027 476e75d115788dcaf63bd61c51a45a2e1f394b68
push id35856
push usercsabou@mozilla.com
push dateFri, 12 Apr 2019 03:19:48 +0000
treeherdermozilla-central@940684cd1065 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato
bugs1536804
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1536804 - Factor out a base class for the wpt metadata compilers, r=ato The manifest compilers follow a mostly common design so it seems reasonable to put the common parts into a base class, and allow subclasses to override the functionaility where required. Differential Revision: https://phabricator.services.mozilla.com/D24177
testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptmanifest/backends/base.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptmanifest/backends/static.py
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
@@ -1,14 +1,14 @@
 import os
 import urlparse
 from collections import deque
 
 from wptmanifest.backends import static
-from wptmanifest.backends.static import ManifestItem
+from wptmanifest.backends.base import ManifestItem
 
 import expected
 
 """Manifest structure used to store expected results of a test.
 
 Each manifest file is represented by an ExpectedManifest that
 has one or more TestNode children, one per test in the manifest.
 Each TestNode has zero or more SubtestNode children, one for each
@@ -193,32 +193,33 @@ def fuzzy_prop(node):
             else:
                 value = arg_values[None].popleft()
             range_values.append(value)
         rv.append((key, tuple(range_values)))
     return rv
 
 
 class ExpectedManifest(ManifestItem):
-    def __init__(self, name, test_path, url_base):
+    def __init__(self, node, test_path, url_base):
         """Object representing all the tests in a particular manifest
 
         :param name: Name of the AST Node associated with this object.
                      Should always be None since this should always be associated with
                      the root node of the AST.
         :param test_path: Path of the test file associated with this manifest.
         :param url_base: Base url for serving the tests in this manifest
         """
+        name = node.data
         if name is not None:
             raise ValueError("ExpectedManifest should represent the root node")
         if test_path is None:
             raise ValueError("ExpectedManifest requires a test path")
         if url_base is None:
             raise ValueError("ExpectedManifest requires a base url")
-        ManifestItem.__init__(self, name)
+        ManifestItem.__init__(self, node)
         self.child_map = {}
         self.test_path = test_path
         self.url_base = url_base
 
     def append(self, child):
         """Add a test to the manifest"""
         ManifestItem.append(self, child)
         self.child_map[child.id] = child
@@ -334,22 +335,22 @@ class DirectoryManifest(ManifestItem):
         return int_prop("lsan-max-stack-depth", self)
 
     @property
     def fuzzy(self):
         return fuzzy_prop(self)
 
 
 class TestNode(ManifestItem):
-    def __init__(self, name):
+    def __init__(self, node, **kwargs):
         """Tree node associated with a particular test in a manifest
 
         :param name: name of the test"""
-        assert name is not None
-        ManifestItem.__init__(self, name)
+        assert node.data is not None
+        ManifestItem.__init__(self, node, **kwargs)
         self.updated_expected = []
         self.new_expected = []
         self.subtests = {}
         self.default_status = None
         self._from_file = True
 
     @property
     def is_empty(self):
@@ -426,22 +427,16 @@ class TestNode(ManifestItem):
 
         :param name: Name of the node to return"""
         if name in self.subtests:
             return self.subtests[name]
         return None
 
 
 class SubtestNode(TestNode):
-    def __init__(self, name):
-        """Tree node associated with a particular subtest in a manifest
-
-        :param name: name of the subtest"""
-        TestNode.__init__(self, name)
-
     @property
     def is_empty(self):
         if self._data:
             return False
         return True
 
 
 def get_manifest(metadata_root, test_path, url_base, run_info):
copy from testing/web-platform/tests/tools/wptrunner/wptrunner/wptmanifest/backends/static.py
copy to testing/web-platform/tests/tools/wptrunner/wptrunner/wptmanifest/backends/base.py
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptmanifest/backends/static.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptmanifest/backends/base.py
@@ -1,142 +1,141 @@
-import operator
+import abc
 
 from ..node import NodeVisitor
 from ..parser import parse
 
 
 class Compiler(NodeVisitor):
-    """Compiler backend that evaluates conditional expressions
-    to give static output"""
+    __metaclass__ = abc.ABCMeta
 
-    def compile(self, tree, expr_data, data_cls_getter=None, **kwargs):
-        """Compile a raw AST into a form with conditional expressions
-        evaluated.
+    def compile(self, tree, data_cls_getter=None, **kwargs):
+        self._kwargs = kwargs
+        return self._compile(tree, data_cls_getter, **kwargs)
+
+    def _compile(self, tree, data_cls_getter=None, **kwargs):
+        """Compile a raw AST into a form where conditional expressions
+        are represented by ConditionalValue objects that can be evaluated
+        at runtime.
 
         tree - The root node of the wptmanifest AST to compile
 
-        expr_data - A dictionary of key / value pairs to use when
-                    evaluating conditional expressions
-
         data_cls_getter - A function taking two parameters; the previous
                           output node and the current ast node and returning
                           the class of the output node to use for the current
                           ast node
         """
-
-        self._kwargs = kwargs
-        self.expr_data = expr_data
-
         if data_cls_getter is None:
             self.data_cls_getter = lambda x, y: ManifestItem
         else:
             self.data_cls_getter = data_cls_getter
 
-        self.output_node = None
+        self.tree = tree
+        self.output_node = self._initial_output_node(tree, **kwargs)
         self.visit(tree)
+        if hasattr(self.output_node, "set_defaults"):
+            self.output_node.set_defaults()
+        assert self.output_node is not None
         return self.output_node
 
+    def _initial_output_node(self, node, **kwargs):
+        return self.data_cls_getter(None, None)(node, **kwargs)
+
     def visit_DataNode(self, node):
-        output_parent = self.output_node
-        if self.output_node is None:
-            assert node.parent is None
-            self.output_node = self.data_cls_getter(None, None)(None, **self._kwargs)
+        if node != self.tree:
+            output_parent = self.output_node
+            self.output_node = self.data_cls_getter(self.output_node, node)(node, **self._kwargs)
         else:
-            self.output_node = self.data_cls_getter(self.output_node, node)(node.data)
+            output_parent = None
+
+        assert self.output_node is not None
 
         for child in node.children:
             self.visit(child)
 
         if output_parent is not None:
+            # Append to the parent *after* processing all the node data
             output_parent.append(self.output_node)
             self.output_node = self.output_node.parent
 
+        assert self.output_node is not None
+
+    @abc.abstractmethod
     def visit_KeyValueNode(self, node):
-        key_name = node.data
-        key_value = None
-        for child in node.children:
-            value = self.visit(child)
-            if value is not None:
-                key_value = value
-                break
-        if key_value is not None:
-            self.output_node.set(key_name, key_value)
+        pass
+
+    def visit_ListNode(self, node):
+        return [self.visit(child) for child in node.children]
 
     def visit_ValueNode(self, node):
         return node.data
 
     def visit_AtomNode(self, node):
         return node.data
 
-    def visit_ListNode(self, node):
-        return [self.visit(child) for child in node.children]
-
+    @abc.abstractmethod
     def visit_ConditionalNode(self, node):
-        assert len(node.children) == 2
-        if self.visit(node.children[0]):
-            return self.visit(node.children[1])
+        pass
 
     def visit_StringNode(self, node):
-        value = node.data
-        for child in node.children:
-            value = self.visit(child)(value)
+        indexes = [self.visit(child) for child in node.children]
+
+        def value(x):
+            rv = node.data
+            for index in indexes:
+                rv = rv[index(x)]
+            return rv
         return value
 
     def visit_NumberNode(self, node):
         if "." in node.data:
             return float(node.data)
         else:
             return int(node.data)
 
     def visit_VariableNode(self, node):
-        value = self.expr_data[node.data]
-        for child in node.children:
-            value = self.visit(child)(value)
+        indexes = [self.visit(child) for child in node.children]
+
+        def value(x):
+            data = x[node.data]
+            for index in indexes:
+                data = data[index(x)]
+            return data
         return value
 
     def visit_IndexNode(self, node):
         assert len(node.children) == 1
-        index = self.visit(node.children[0])
-        return lambda x: x[index]
+        return self.visit(node.children[0])
 
+    @abc.abstractmethod
     def visit_UnaryExpressionNode(self, node):
-        assert len(node.children) == 2
-        operator = self.visit(node.children[0])
-        operand = self.visit(node.children[1])
+        pass
 
-        return operator(operand)
-
+    @abc.abstractmethod
     def visit_BinaryExpressionNode(self, node):
-        assert len(node.children) == 3
-        operator = self.visit(node.children[0])
-        operand_0 = self.visit(node.children[1])
-        operand_1 = self.visit(node.children[2])
+        pass
 
-        return operator(operand_0, operand_1)
-
+    @abc.abstractmethod
     def visit_UnaryOperatorNode(self, node):
-        return {"not": operator.not_}[node.data]
+        pass
 
+    @abc.abstractmethod
     def visit_BinaryOperatorNode(self, node):
-        return {"and": operator.and_,
-                "or": operator.or_,
-                "==": operator.eq,
-                "!=": operator.ne}[node.data]
+        pass
 
 
 class ManifestItem(object):
-    def __init__(self, name, **kwargs):
+    def __init__(self, node, **kwargs):
         self.parent = None
-        self.name = name
+        self.node = node
         self.children = []
         self._data = {}
 
     def __repr__(self):
-        return "<ManifestItem %s>" % (self.name)
+        return "<ManifestItem %s>" % (self.node.data)
 
     def __str__(self):
         rv = [repr(self)]
         for item in self.children:
             rv.extend("  %s" % line for line in str(item).split("\n"))
         return "\n".join(rv)
 
     def set_defaults(self):
@@ -150,44 +149,45 @@ class ManifestItem(object):
 
     @property
     def root(self):
         node = self
         while node.parent is not None:
             node = node.parent
         return node
 
-    def has_key(self, key):
-        for node in [self, self.root]:
-            if key in node._data:
-                return True
-        return False
+    @property
+    def name(self):
+        return self.node.data
 
     def get(self, key):
         for node in [self, self.root]:
             if key in node._data:
                 return node._data[key]
         raise KeyError
 
     def set(self, name, value):
         self._data[name] = value
 
     def remove(self):
         if self.parent:
-            self.parent._remove_child(self)
-
-    def _remove_child(self, child):
-        self.children.remove(child)
-        child.parent = None
+            self.parent.children.remove(child)
+            self.parent = None
 
     def iterchildren(self, name=None):
         for item in self.children:
             if item.name == name or name is None:
                 yield item
 
+    def has_key(self, key):
+        for node in [self, self.root]:
+            if key in node._data:
+                return True
+        return False
+
     def _flatten(self):
         rv = {}
         for node in [self, self.root]:
             for name, value in node._data.iteritems():
                 if name not in rv:
                     rv[name] = value
         return rv
 
@@ -204,20 +204,19 @@ class ManifestItem(object):
             yield item
 
     def append(self, child):
         child.parent = self
         self.children.append(child)
         return child
 
 
-def compile_ast(ast, expr_data, data_cls_getter=None, **kwargs):
-    return Compiler().compile(ast,
-                              expr_data,
+def compile_ast(compiler, ast, data_cls_getter=None, **kwargs):
+    return compiler().compile(ast,
                               data_cls_getter=data_cls_getter,
                               **kwargs)
 
 
-def compile(stream, expr_data, data_cls_getter=None, **kwargs):
-    return compile_ast(parse(stream),
-                       expr_data,
+def compile(compiler, stream, data_cls_getter=None, **kwargs):
+    return compile_ast(compiler,
+                       parse(stream),
                        data_cls_getter=data_cls_getter,
                        **kwargs)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptmanifest/backends/static.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptmanifest/backends/static.py
@@ -1,15 +1,15 @@
 import operator
 
-from ..node import NodeVisitor
+import base
 from ..parser import parse
 
 
-class Compiler(NodeVisitor):
+class Compiler(base.Compiler):
     """Compiler backend that evaluates conditional expressions
     to give static output"""
 
     def compile(self, tree, expr_data, data_cls_getter=None, **kwargs):
         """Compile a raw AST into a form with conditional expressions
         evaluated.
 
         tree - The root node of the wptmanifest AST to compile
@@ -21,88 +21,46 @@ class Compiler(NodeVisitor):
                           output node and the current ast node and returning
                           the class of the output node to use for the current
                           ast node
         """
 
         self._kwargs = kwargs
         self.expr_data = expr_data
 
-        if data_cls_getter is None:
-            self.data_cls_getter = lambda x, y: ManifestItem
-        else:
-            self.data_cls_getter = data_cls_getter
-
-        self.output_node = None
-        self.visit(tree)
-        return self.output_node
-
-    def visit_DataNode(self, node):
-        output_parent = self.output_node
-        if self.output_node is None:
-            assert node.parent is None
-            self.output_node = self.data_cls_getter(None, None)(None, **self._kwargs)
-        else:
-            self.output_node = self.data_cls_getter(self.output_node, node)(node.data)
-
-        for child in node.children:
-            self.visit(child)
-
-        if output_parent is not None:
-            output_parent.append(self.output_node)
-            self.output_node = self.output_node.parent
+        return self._compile(tree, data_cls_getter, **kwargs)
 
     def visit_KeyValueNode(self, node):
         key_name = node.data
         key_value = None
         for child in node.children:
             value = self.visit(child)
             if value is not None:
                 key_value = value
                 break
         if key_value is not None:
             self.output_node.set(key_name, key_value)
 
-    def visit_ValueNode(self, node):
-        return node.data
-
-    def visit_AtomNode(self, node):
-        return node.data
-
-    def visit_ListNode(self, node):
-        return [self.visit(child) for child in node.children]
-
     def visit_ConditionalNode(self, node):
         assert len(node.children) == 2
         if self.visit(node.children[0]):
             return self.visit(node.children[1])
 
     def visit_StringNode(self, node):
         value = node.data
         for child in node.children:
             value = self.visit(child)(value)
         return value
 
-    def visit_NumberNode(self, node):
-        if "." in node.data:
-            return float(node.data)
-        else:
-            return int(node.data)
-
     def visit_VariableNode(self, node):
         value = self.expr_data[node.data]
         for child in node.children:
             value = self.visit(child)(value)
         return value
 
-    def visit_IndexNode(self, node):
-        assert len(node.children) == 1
-        index = self.visit(node.children[0])
-        return lambda x: x[index]
-
     def visit_UnaryExpressionNode(self, node):
         assert len(node.children) == 2
         operator = self.visit(node.children[0])
         operand = self.visit(node.children[1])
 
         return operator(operand)
 
     def visit_BinaryExpressionNode(self, node):
@@ -118,102 +76,16 @@ class Compiler(NodeVisitor):
 
     def visit_BinaryOperatorNode(self, node):
         return {"and": operator.and_,
                 "or": operator.or_,
                 "==": operator.eq,
                 "!=": operator.ne}[node.data]
 
 
-class ManifestItem(object):
-    def __init__(self, name, **kwargs):
-        self.parent = None
-        self.name = name
-        self.children = []
-        self._data = {}
-
-    def __repr__(self):
-        return "<ManifestItem %s>" % (self.name)
-
-    def __str__(self):
-        rv = [repr(self)]
-        for item in self.children:
-            rv.extend("  %s" % line for line in str(item).split("\n"))
-        return "\n".join(rv)
-
-    def set_defaults(self):
-        pass
-
-    @property
-    def is_empty(self):
-        if self._data:
-            return False
-        return all(child.is_empty for child in self.children)
-
-    @property
-    def root(self):
-        node = self
-        while node.parent is not None:
-            node = node.parent
-        return node
-
-    def has_key(self, key):
-        for node in [self, self.root]:
-            if key in node._data:
-                return True
-        return False
-
-    def get(self, key):
-        for node in [self, self.root]:
-            if key in node._data:
-                return node._data[key]
-        raise KeyError
-
-    def set(self, name, value):
-        self._data[name] = value
-
-    def remove(self):
-        if self.parent:
-            self.parent._remove_child(self)
-
-    def _remove_child(self, child):
-        self.children.remove(child)
-        child.parent = None
-
-    def iterchildren(self, name=None):
-        for item in self.children:
-            if item.name == name or name is None:
-                yield item
-
-    def _flatten(self):
-        rv = {}
-        for node in [self, self.root]:
-            for name, value in node._data.iteritems():
-                if name not in rv:
-                    rv[name] = value
-        return rv
-
-    def iteritems(self):
-        for item in self._flatten().iteritems():
-            yield item
-
-    def iterkeys(self):
-        for item in self._flatten().iterkeys():
-            yield item
-
-    def itervalues(self):
-        for item in self._flatten().itervalues():
-            yield item
-
-    def append(self, child):
-        child.parent = self
-        self.children.append(child)
-        return child
-
-
 def compile_ast(ast, expr_data, data_cls_getter=None, **kwargs):
     return Compiler().compile(ast,
                               expr_data,
                               data_cls_getter=data_cls_getter,
                               **kwargs)
 
 
 def compile(stream, expr_data, data_cls_getter=None, **kwargs):