Bug 1340551 - [mozlog] Introduce concept of ContainerType in logtypes and allow nested containers, r=jgraham
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 21 Feb 2017 14:24:14 -0500
changeset 394141 0f5ee692c95699047d89373391b9f81ed269873d
parent 394140 6c1627e917136d72f3b7d74da2d10d45315dd929
child 394142 9faf01462771e126b184df16ce18b4c18ef61465
push id1468
push userasasaki@mozilla.com
push dateMon, 05 Jun 2017 19:31:07 +0000
treeherdermozilla-release@0641fc6ee9d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1340551
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1340551 - [mozlog] Introduce concept of ContainerType in logtypes and allow nested containers, r=jgraham Currently the List and Tuple DataTypes must specify what items they contain. But there's no way to specify item types recursively, e.g List(Tuple(Int, Int)). Also the Dict type can't specify the item types it must contain either. Dict is a special case because we may want to control both keys and values. This patch formalizes a ContainerType (of which List, Tuple and Dict are subclasses). MozReview-Commit-ID: Bouhy1DIAyD
testing/mozbase/mozlog/mozlog/logtypes.py
testing/mozbase/mozlog/mozlog/structuredlog.py
testing/mozbase/mozlog/tests/manifest.ini
testing/mozbase/mozlog/tests/test_logtypes.py
--- a/testing/mozbase/mozlog/mozlog/logtypes.py
+++ b/testing/mozbase/mozlog/mozlog/logtypes.py
@@ -1,12 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import inspect
+
 convertor_registry = {}
 missing = object()
 no_default = object()
 
 
 class log_action(object):
 
     def __init__(self, *args):
@@ -118,16 +120,41 @@ class DataType(object):
 
         try:
             return self.convert(value)
         except:
             raise ValueError("Failed to convert value %s of type %s for field %s to type %s" %
                              (value, type(value).__name__, self.name, self.__class__.__name__))
 
 
+class ContainerType(DataType):
+    """A DataType that contains other DataTypes.
+
+    ContainerTypes must specify which other DataType they will contain. ContainerTypes
+    may contain other ContainerTypes.
+
+    Some examples:
+
+        List(Int, 'numbers')
+        Tuple((Unicode, Int, Any), 'things')
+        Dict(Unicode, 'config')
+        Dict({TestId: Status}, 'results')
+        Dict(List(Unicode), 'stuff')
+    """
+
+    def __init__(self, item_type, name=None, **kwargs):
+        DataType.__init__(self, name, **kwargs)
+        self.item_type = self._format_item_type(item_type)
+
+    def _format_item_type(self, item_type):
+        if inspect.isclass(item_type):
+            return item_type(None)
+        return item_type
+
+
 class Unicode(DataType):
 
     def convert(self, data):
         if isinstance(data, unicode):
             return data
         if isinstance(data, str):
             return data.decode("utf8", "replace")
         return unicode(data)
@@ -158,47 +185,60 @@ class Status(DataType):
             raise ValueError
         return value
 
 
 class SubStatus(Status):
     allowed = ["PASS", "FAIL", "ERROR", "TIMEOUT", "ASSERT", "NOTRUN", "SKIP"]
 
 
-class Dict(DataType):
+class Dict(ContainerType):
+
+    def _format_item_type(self, item_type):
+        superfmt = super(Dict, self)._format_item_type
+
+        if isinstance(item_type, dict):
+            if len(item_type) != 1:
+                raise ValueError("Dict item type specifier must contain a single entry.")
+            key_type, value_type = item_type.items()[0]
+            return superfmt(key_type), superfmt(value_type)
+        return Any(None), superfmt(item_type)
 
     def convert(self, data):
-        return dict(data)
+        key_type, value_type = self.item_type
+        return {key_type.convert(k): value_type.convert(v) for k, v in dict(data).items()}
 
 
-class List(DataType):
-
-    def __init__(self, name, item_type, default=no_default, optional=False):
-        DataType.__init__(self, name, default, optional)
-        self.item_type = item_type(None)
+class List(ContainerType):
 
     def convert(self, data):
+        # while dicts and strings _can_ be cast to lists, doing so is probably not intentional
+        if isinstance(data, (basestring, dict)):
+            raise ValueError("Expected list but got %s" % type(data))
         return [self.item_type.convert(item) for item in data]
 
 
 class Int(DataType):
 
     def convert(self, data):
         return int(data)
 
 
 class Any(DataType):
 
     def convert(self, data):
         return data
 
 
-class Tuple(DataType):
+class Tuple(ContainerType):
 
-    def __init__(self, name, item_types, default=no_default, optional=False):
-        DataType.__init__(self, name, default, optional)
-        self.item_types = item_types
+    def _format_item_type(self, item_type):
+        superfmt = super(Tuple, self)._format_item_type
+
+        if isinstance(item_type, (tuple, list)):
+            return [superfmt(t) for t in item_type]
+        return (superfmt(item_type),)
 
     def convert(self, data):
-        if len(data) != len(self.item_types):
-            raise ValueError("Expected %i items got %i" % (len(self.item_types), len(data)))
+        if len(data) != len(self.item_type):
+            raise ValueError("Expected %i items got %i" % (len(self.item_type), len(data)))
         return tuple(item_type.convert(value)
-                     for item_type, value in zip(self.item_types, data))
+                     for item_type, value in zip(self.item_type, data))
--- a/testing/mozbase/mozlog/mozlog/structuredlog.py
+++ b/testing/mozbase/mozlog/mozlog/structuredlog.py
@@ -251,36 +251,36 @@ class StructuredLogger(object):
         elif action == 'suite_end':
             if not self._state.suite_started:
                 self.error("Got suite_end message before suite_start. " +
                            "Logged with data: {}".format(json.dumps(data)))
                 return False
             self._state.suite_started = False
         return True
 
-    @log_action(List("tests", Unicode),
-                Dict("run_info", default=None, optional=True),
-                Dict("version_info", default=None, optional=True),
-                Dict("device_info", default=None, optional=True),
-                Dict("extra", default=None, optional=True))
+    @log_action(List(Unicode, "tests"),
+                Dict(Any, "run_info", default=None, optional=True),
+                Dict(Any, "version_info", default=None, optional=True),
+                Dict(Any, "device_info", default=None, optional=True),
+                Dict(Any, "extra", default=None, optional=True))
     def suite_start(self, data):
         """Log a suite_start message
 
         :param list tests: Test identifiers that will be run in the suite.
         :param dict run_info: Optional information typically provided by mozinfo.
         :param dict version_info: Optional target application version information provided
           by mozversion.
         :param dict device_info: Optional target device information provided by mozdevice.
         """
         if not self._ensure_suite_state('suite_start', data):
             return
 
         self._log_data("suite_start", data)
 
-    @log_action(Dict("extra", default=None, optional=True))
+    @log_action(Dict(Any, "extra", default=None, optional=True))
     def suite_end(self, data):
         """Log a suite_end message"""
         if not self._ensure_suite_state('suite_end', data):
             return
 
         self._log_data("suite_end")
 
     @log_action(TestId("test"),
@@ -304,17 +304,17 @@ class StructuredLogger(object):
         self._log_data("test_start", data)
 
     @log_action(TestId("test"),
                 Unicode("subtest"),
                 SubStatus("status"),
                 SubStatus("expected", default="PASS"),
                 Unicode("message", default=None, optional=True),
                 Unicode("stack", default=None, optional=True),
-                Dict("extra", default=None, optional=True))
+                Dict(Any, "extra", default=None, optional=True))
     def test_status(self, data):
         """
         Log a test_status message indicating a subtest result. Tests that
         do not have subtests are not expected to produce test_status messages.
 
         :param test: Identifier of the test that produced the result.
         :param subtest: Name of the subtest.
         :param status: Status string indicating the subtest result
@@ -335,17 +335,17 @@ class StructuredLogger(object):
 
         self._log_data("test_status", data)
 
     @log_action(TestId("test"),
                 Status("status"),
                 Status("expected", default="OK"),
                 Unicode("message", default=None, optional=True),
                 Unicode("stack", default=None, optional=True),
-                Dict("extra", default=None, optional=True))
+                Dict(Any, "extra", default=None, optional=True))
     def test_end(self, data):
         """
         Log a test_end message indicating that a test completed. For tests
         with subtests this indicates whether the overall test completed without
         errors. For tests without subtests this indicates the test result
         directly.
 
         :param test: Identifier of the test that produced the result.
@@ -384,25 +384,25 @@ class StructuredLogger(object):
     @log_action(Unicode("process", default=None),
                 Unicode("signature", default="[Unknown]"),
                 TestId("test", default=None, optional=True),
                 Unicode("minidump_path", default=None, optional=True),
                 Unicode("minidump_extra", default=None, optional=True),
                 Int("stackwalk_retcode", default=None, optional=True),
                 Unicode("stackwalk_stdout", default=None, optional=True),
                 Unicode("stackwalk_stderr", default=None, optional=True),
-                List("stackwalk_errors", Unicode, default=None))
+                List(Unicode, "stackwalk_errors", default=None))
     def crash(self, data):
         if data["stackwalk_errors"] is None:
             data["stackwalk_errors"] = []
 
         self._log_data("crash", data)
 
     @log_action(Unicode("primary", default=None),
-                List("secondary", Unicode, default=None))
+                List(Unicode, "secondary", default=None))
     def valgrind_error(self, data):
         self._log_data("valgrind_error", data)
 
     @log_action(Unicode("process"),
                 Unicode("command", default=None, optional=True))
     def process_start(self, data):
         """Log start event of a process.
 
@@ -471,17 +471,17 @@ def _log_func(level_name):
 def _lint_func(level_name):
     @log_action(Unicode("path"),
                 Unicode("message", default=""),
                 Int("lineno", default=0),
                 Int("column", default=None, optional=True),
                 Unicode("hint", default=None, optional=True),
                 Unicode("source", default=None, optional=True),
                 Unicode("rule", default=None, optional=True),
-                Tuple("lineoffset", (Int, Int), default=None, optional=True),
+                Tuple((Int, Int), "lineoffset", default=None, optional=True),
                 Unicode("linter", default=None, optional=True))
     def lint(self, data):
         data["level"] = level_name
         self._log_data("lint", data)
     lint.__doc__ = """Log an error resulting from a failed lint check
 
         :param linter: name of the linter that flagged this error
         :param path: path to the file containing the error
--- a/testing/mozbase/mozlog/tests/manifest.ini
+++ b/testing/mozbase/mozlog/tests/manifest.ini
@@ -1,4 +1,5 @@
 [DEFAULT]
 subsuite = mozbase, os == "linux"
 [test_logger.py]
+[test_logtypes.py]
 [test_structured.py]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozlog/tests/test_logtypes.py
@@ -0,0 +1,99 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+import mozunit
+
+from mozlog.logtypes import (
+    Any,
+    Dict,
+    Int,
+    List,
+    Tuple,
+    Unicode,
+)
+
+
+class TestContainerTypes(unittest.TestCase):
+
+    def test_dict_type_basic(self):
+        d = Dict('name')
+        with self.assertRaises(ValueError):
+            d({'foo': 'bar'})
+
+        d = Dict(Any, 'name')
+        d({'foo': 'bar'})  # doesn't raise
+
+    def test_dict_type_with_dictionary_item_type(self):
+        d = Dict({Int: Int}, 'name')
+        with self.assertRaises(ValueError):
+            d({'foo': 1})
+
+        with self.assertRaises(ValueError):
+            d({1: 'foo'})
+
+        d({1: 2})  # doesn't raise
+
+    def test_dict_type_with_recursive_item_types(self):
+        d = Dict(Dict({Unicode: List(Int)}), 'name')
+        with self.assertRaises(ValueError):
+            d({'foo': 'bar'})
+
+        with self.assertRaises(ValueError):
+            d({'foo': {'bar': 'baz'}})
+
+        with self.assertRaises(ValueError):
+            d({'foo': {'bar': ['baz']}})
+
+        d({'foo': {'bar': [1]}})  # doesn't raise
+
+    def test_list_type_basic(self):
+        l = List('name')
+        with self.assertRaises(ValueError):
+            l(['foo'])
+
+        l = List(Any, 'name')
+        l(['foo', 1])  # doesn't raise
+
+    def test_list_type_with_recursive_item_types(self):
+        l = List(Dict(List(Tuple((Unicode, Int)))), 'name')
+        with self.assertRaises(ValueError):
+            l(['foo'])
+
+        with self.assertRaises(ValueError):
+            l([{'foo': 'bar'}])
+
+        with self.assertRaises(ValueError):
+            l([{'foo': ['bar']}])
+
+        l([{'foo': [('bar', 1)]}])  # doesn't raise
+
+    def test_tuple_type_basic(self):
+        t = Tuple('name')
+        with self.assertRaises(ValueError):
+            t((1,))
+
+        t = Tuple(Any, 'name')
+        t((1,))  # doesn't raise
+
+    def test_tuple_type_with_tuple_item_type(self):
+        t = Tuple((Unicode, Int))
+        with self.assertRaises(ValueError):
+            t(('foo', 'bar'))
+
+        t(('foo', 1))  # doesn't raise
+
+    def test_tuple_type_with_recursive_item_types(self):
+        t = Tuple((Dict(List(Any)), List(Dict(Any)), Unicode), 'name')
+        with self.assertRaises(ValueError):
+            t(({'foo': 'bar'}, [{'foo': 'bar'}], 'foo'))
+
+        with self.assertRaises(ValueError):
+            t(({'foo': ['bar']}, ['foo'], 'foo'))
+
+        t(({'foo': ['bar']}, [{'foo': 'bar'}], 'foo'))  # doesn't raise
+
+
+if __name__ == '__main__':
+    mozunit.main()