Bug 1340551 - [mozlog] Change 'tests' field in suite_start action to a dict of tests keyed by group name, r=jgraham
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 22 Feb 2017 11:46:09 -0500
changeset 394142 9faf01462771e126b184df16ce18b4c18ef61465
parent 394141 0f5ee692c95699047d89373391b9f81ed269873d
child 394143 1fb190001b3ecc45514688c5628239cc2e221fc8
push id1468
push userasasaki@mozilla.com
push dateMon, 05 Jun 2017 19:31:07 +0000
treeherdermozilla-release@0641fc6ee9d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1340551
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1340551 - [mozlog] Change 'tests' field in suite_start action to a dict of tests keyed by group name, r=jgraham In suite_start, tests is currently a list of test ids. But to support a new 'test-centric' treeherder view, we need to be able to map tests to their manifests somewhere in the structured log, and the suite_start tests field seemed like a good spot. Because not all test suites use manifests, this is being called a "group" so it could potentially be re-used with directories, tags or subsuites. MozReview-Commit-ID: 2Sc7nqJrWts
testing/mozbase/docs/mozlog.rst
testing/mozbase/mozlog/mozlog/formatters/machformatter.py
testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
testing/mozbase/mozlog/mozlog/logtypes.py
testing/mozbase/mozlog/mozlog/structuredlog.py
testing/mozbase/mozlog/tests/test_logtypes.py
testing/mozbase/mozlog/tests/test_structured.py
--- a/testing/mozbase/docs/mozlog.rst
+++ b/testing/mozbase/docs/mozlog.rst
@@ -58,21 +58,25 @@ on on all messages is:
 For each ``action`` there are is a further set of specific fields
 describing the details of the event that caused the message to be
 emitted:
 
 ``suite_start``
   Emitted when the testsuite starts running.
 
   ``tests``
-    A list of test ids. Test ids can either be strings or lists of
-    strings (an example of the latter is reftests where the id has the
-    form [test_url, ref_type, ref_url]) and are assumed to be unique
-    within a given testsuite. In cases where the test list is not
-    known upfront an empty list may be passed (list).
+    A dict of test ids keyed by group. Groups are any logical grouping
+    of tests, for example a manifest, directory or tag. For convenience,
+    a list of test ids can be used instead. In this case all tests will
+    automatically be placed in the 'default' group name. Test ids can
+    either be strings or lists of strings (an example of the latter is
+    reftests where the id has the form [test_url, ref_type, ref_url]).
+    Test ids are assumed to be unique within a given testsuite. In cases
+    where the test list is not known upfront an empty dict or list may
+    be passed (dict).
 
   ``run_info``
     An optional dictionary describing the properties of the
     build and test environment. This contains the information provided
     by :doc:`mozinfo <mozinfo>`, plus a boolean ``debug`` field indicating
     whether the build under test is a debug build.
 
 ``suite_end``
--- a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
@@ -116,17 +116,18 @@ class MachFormatter(base.BaseFormatter):
     def suite_start(self, data):
         self.summary_values = {"tests": 0,
                                "subtests": 0,
                                "assertion_counts": 0,
                                "expected": 0,
                                "unexpected": defaultdict(int),
                                "skipped": 0}
         self.summary_unexpected = []
-        return "%i" % len(data["tests"])
+        num_tests = reduce(lambda x, y: x + len(y), data['tests'].itervalues(), 0)
+        return "%i" % num_tests
 
     def suite_end(self, data):
         term = self.terminal if self.terminal is not None else NullTerminal()
 
         heading = "Summary"
         rv = ["", heading, "=" * len(heading), ""]
 
         has_subtests = self.summary_values["subtests"] > 0
--- a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
@@ -112,17 +112,18 @@ class TbplFormatter(BaseFormatter):
         rv = "\n".join(rv)
         if not rv[-1] == "\n":
             rv += "\n"
 
         return rv
 
     def suite_start(self, data):
         self.suite_start_time = data["time"]
-        return "SUITE-START | Running %i tests\n" % len(data["tests"])
+        num_tests = reduce(lambda x, y: x + len(y), data['tests'].itervalues(), 0)
+        return "SUITE-START | Running %i tests\n" % num_tests
 
     def test_start(self, data):
         self.test_start_times[self.test_id(data["test"])] = data["time"]
 
         return "TEST-START | %s\n" % data["test"]
 
     def test_status(self, data):
         if self.compact:
--- a/testing/mozbase/mozlog/mozlog/logtypes.py
+++ b/testing/mozbase/mozlog/mozlog/logtypes.py
@@ -205,22 +205,34 @@ class Dict(ContainerType):
     def convert(self, data):
         key_type, value_type = self.item_type
         return {key_type.convert(k): value_type.convert(v) for k, v in dict(data).items()}
 
 
 class List(ContainerType):
 
     def convert(self, data):
-        # while dicts and strings _can_ be cast to lists, doing so is probably not intentional
+        # while dicts and strings _can_ be cast to lists,
+        # doing so is likely not intentional behaviour
         if isinstance(data, (basestring, dict)):
             raise ValueError("Expected list but got %s" % type(data))
         return [self.item_type.convert(item) for item in data]
 
 
+class TestList(DataType):
+    """A TestList is a list of tests that can be either keyed by a group name,
+    or specified as a flat list.
+    """
+
+    def convert(self, data):
+        if isinstance(data, (list, tuple)):
+            data = {'default': data}
+        return Dict({Unicode: List(Unicode)}).convert(data)
+
+
 class Int(DataType):
 
     def convert(self, data):
         return int(data)
 
 
 class Any(DataType):
 
--- a/testing/mozbase/mozlog/mozlog/structuredlog.py
+++ b/testing/mozbase/mozlog/mozlog/structuredlog.py
@@ -6,17 +6,17 @@ from __future__ import unicode_literals
 
 from multiprocessing import current_process
 from threading import current_thread, Lock
 import json
 import sys
 import time
 import traceback
 
-from logtypes import Unicode, TestId, Status, SubStatus, Dict, List, Int, Any, Tuple
+from logtypes import Unicode, TestId, TestList, Status, SubStatus, Dict, List, Int, Any, Tuple
 from logtypes import log_action, convertor_registry
 
 """Structured Logging for recording test results.
 
 Allowed actions, and subfields:
   suite_start
       tests  - List of test names
 
@@ -251,25 +251,25 @@ class StructuredLogger(object):
         elif action == 'suite_end':
             if not self._state.suite_started:
                 self.error("Got suite_end message before suite_start. " +
                            "Logged with data: {}".format(json.dumps(data)))
                 return False
             self._state.suite_started = False
         return True
 
-    @log_action(List(Unicode, "tests"),
+    @log_action(TestList("tests"),
                 Dict(Any, "run_info", default=None, optional=True),
                 Dict(Any, "version_info", default=None, optional=True),
                 Dict(Any, "device_info", default=None, optional=True),
                 Dict(Any, "extra", default=None, optional=True))
     def suite_start(self, data):
         """Log a suite_start message
 
-        :param list tests: Test identifiers that will be run in the suite.
+        :param dict tests: Test identifiers that will be run in the suite, keyed by group name.
         :param dict run_info: Optional information typically provided by mozinfo.
         :param dict version_info: Optional target application version information provided
           by mozversion.
         :param dict device_info: Optional target device information provided by mozdevice.
         """
         if not self._ensure_suite_state('suite_start', data):
             return
 
--- a/testing/mozbase/mozlog/tests/test_logtypes.py
+++ b/testing/mozbase/mozlog/tests/test_logtypes.py
@@ -5,16 +5,17 @@
 import unittest
 import mozunit
 
 from mozlog.logtypes import (
     Any,
     Dict,
     Int,
     List,
+    TestList,
     Tuple,
     Unicode,
 )
 
 
 class TestContainerTypes(unittest.TestCase):
 
     def test_dict_type_basic(self):
@@ -90,10 +91,26 @@ class TestContainerTypes(unittest.TestCa
             t(({'foo': 'bar'}, [{'foo': 'bar'}], 'foo'))
 
         with self.assertRaises(ValueError):
             t(({'foo': ['bar']}, ['foo'], 'foo'))
 
         t(({'foo': ['bar']}, [{'foo': 'bar'}], 'foo'))  # doesn't raise
 
 
+class TestDataTypes(unittest.TestCase):
+
+    def test_test_list(self):
+        t = TestList('name')
+        with self.assertRaises(ValueError):
+            t('foo')
+
+        with self.assertRaises(ValueError):
+            t({'foo': 1})
+
+        d1 = t({'default': ['bar']})  # doesn't raise
+        d2 = t(['bar'])  # doesn't raise
+
+        self.assertDictContainsSubset(d1, d2)
+
+
 if __name__ == '__main__':
     mozunit.main()
--- a/testing/mozbase/mozlog/tests/test_structured.py
+++ b/testing/mozbase/mozlog/tests/test_structured.py
@@ -105,17 +105,17 @@ class TestStatusHandler(BaseStructuredTe
         self.assertEqual(2, summary.expected_statuses['OK'])
 
 
 class TestStructuredLog(BaseStructuredTest):
 
     def test_suite_start(self):
         self.logger.suite_start(["test"])
         self.assert_log_equals({"action": "suite_start",
-                                "tests": ["test"]})
+                                "tests": {"default": ["test"]}})
         self.logger.suite_end()
 
     def test_suite_end(self):
         self.logger.suite_start([])
         self.logger.suite_end()
         self.assert_log_equals({"action": "suite_end"})
 
     def test_start(self):
@@ -258,27 +258,27 @@ class TestStructuredLog(BaseStructuredTe
         self.assertEquals(last_item["level"], "ERROR")
         self.assertTrue(last_item["message"].startswith(
             "test_end for test2 logged while not in progress. Logged with data: {"))
         self.logger.suite_end()
 
     def test_suite_start_twice(self):
         self.logger.suite_start([])
         self.assert_log_equals({"action": "suite_start",
-                                "tests": []})
+                                "tests": {"default": []}})
         self.logger.suite_start([])
         last_item = self.pop_last_item()
         self.assertEquals(last_item["action"], "log")
         self.assertEquals(last_item["level"], "ERROR")
         self.logger.suite_end()
 
     def test_suite_end_no_start(self):
         self.logger.suite_start([])
         self.assert_log_equals({"action": "suite_start",
-                                "tests": []})
+                                "tests": {"default": []}})
         self.logger.suite_end()
         self.assert_log_equals({"action": "suite_end"})
         self.logger.suite_end()
         last_item = self.pop_last_item()
         self.assertEquals(last_item["action"], "log")
         self.assertEquals(last_item["level"], "ERROR")
 
     def test_multiple_loggers_suite_start(self):
@@ -396,17 +396,17 @@ class TestStructuredLog(BaseStructuredTe
 
 class TestTypeConversions(BaseStructuredTest):
 
     def test_raw(self):
         self.logger.log_raw({"action": "suite_start",
                              "tests": [1],
                              "time": "1234"})
         self.assert_log_equals({"action": "suite_start",
-                                "tests": ["1"],
+                                "tests": {"default": ["1"]},
                                 "time": 1234})
         self.logger.suite_end()
 
     def test_tuple(self):
         self.logger.suite_start([])
         self.logger.test_start(("\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90\x8d\x83\xf0\x90\x8d\x84",
                                 42, u"\u16a4"))
         self.assert_log_equals({"action": "test_start",
@@ -441,17 +441,17 @@ class TestTypeConversions(BaseStructured
     def test_arguments(self):
         self.logger.info(message="test")
         self.assert_log_equals({"action": "log",
                                 "message": "test",
                                 "level": "INFO"})
 
         self.logger.suite_start([], {})
         self.assert_log_equals({"action": "suite_start",
-                                "tests": [],
+                                "tests": {"default": []},
                                 "run_info": {}})
         self.logger.test_start(test="test1")
         self.logger.test_status(
             "subtest1", "FAIL", test="test1", status="PASS")
         self.assert_log_equals({"action": "test_status",
                                 "test": "test1",
                                 "subtest": "subtest1",
                                 "status": "PASS",
@@ -1007,17 +1007,17 @@ class TestBuffer(BaseStructuredTest):
                                 "test": "test1",
                                 "status": "PASS",
                                 "subtest": "sub6"})
         self.assert_log_equals({"action": "test_status",
                                 "test": "test1",
                                 "status": "PASS",
                                 "subtest": "sub5"})
         self.assert_log_equals({"action": "suite_start",
-                                "tests": []})
+                                "tests": {"default": []}})
 
 
 class TestReader(unittest.TestCase):
 
     def to_file_like(self, obj):
         data_str = "\n".join(json.dumps(item) for item in obj)
         return StringIO.StringIO(data_str)