Bug 1204625 - Add series data (unit, whether lower is better) to Talos output;r=jmaher, a=test-only
authorWilliam Lachance <wlachance@mozilla.com>
Fri, 30 Oct 2015 17:26:39 -0400
changeset 305749 3b6903259d21ce7766c4e33bd0c5467e67d2ccf9
parent 305748 8036c2f9c76d73ff6496dabc93e4ed1319ba3222
child 305750 e83bb92f73cfd6b86e4b56b88027cb3ef79f0c16
push id1001
push userraliiev@mozilla.com
push dateMon, 18 Jan 2016 19:06:03 +0000
treeherdermozilla-release@8b89261f3ac4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher, test-only
bugs1204625
milestone44.0a2
Bug 1204625 - Add series data (unit, whether lower is better) to Talos output;r=jmaher, a=test-only
testing/talos/talos/output.py
testing/talos/talos/test.py
--- a/testing/talos/talos/output.py
+++ b/testing/talos/talos/output.py
@@ -512,21 +512,27 @@ class PerfherderOutput(Output):
                 tresults = [tsresult] if tsresult else test.results
 
                 for result in tresults:
                     filtered_results = \
                         result.values(test_result['testrun']['suite'],
                                       test.test_config['filters'])
                     vals.extend([[i['value'], j] for i, j in filtered_results])
                     for val, page in filtered_results:
+                        measurement_metadata = {}
+                        if test.test_config.get('lower_is_better') is not None:
+                            measurement_metadata['lowerIsBetter'] = test.test_config['lower_is_better']
+                        if test.test_config.get('unit'):
+                            measurement_metadata['unit'] = test.test_config['unit']
                         if page == 'NULL':
                             summary['subtests'][test.name()] = val
+                            summary['subtests'][test.name()].update(measurement_metadata)
                         else:
                             summary['subtests'][page] = val
-
+                            summary['subtests'][page].update(measurement_metadata)
 
                 suite_summary = self.construct_results(vals,
                                                        testname=test.name())
                 summary['suite'] = suite_summary
                 test_result['summary'] = summary
 
                 for result, values in results.items():
                     test_result['results'][result] = values
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -25,16 +25,17 @@ def test_dict():
 
 
 class Test(object):
     """abstract base class for a Talos test case"""
     cycles = None  # number of cycles
     keys = []
     desktop = True
     filters = filter.ignore_first.prepare(1) + filter.median.prepare()
+    lower_is_better = True
 
     @classmethod
     def name(cls):
         return cls.__name__
 
     @classmethod
     def description(cls):
         if cls.__doc__ is None:
@@ -132,16 +133,17 @@ class ts_paint(TsBase):
     shutdown = False
     xperf_counters = []
     win7_counters = []
     filters = filter.ignore_first.prepare(1) + filter.median.prepare()
     tpmozafterpaint = True
     rss = False
     mainthread = False
     responsiveness = False
+    unit = 'ms'
 
 
 @register_test()
 class sessionrestore(TsBase):
     """
     A start up test measuring the time it takes to load a sessionstore.js file.
 
     1. Set up Firefox to restore from a given sessionstore.js file.
@@ -154,16 +156,17 @@ class sessionrestore(TsBase):
     sps_profile_startup = True
     sps_profile_entries = 10000000
     profile_path = '${talos}/startup_test/sessionrestore/profile'
     url = 'startup_test/sessionrestore/index.html'
     shutdown = False
     reinstall = ['sessionstore.js', 'sessionCheckpoints.json']
     # Restore the session
     preferences = {'browser.startup.page': 3}
+    unit = 'ms'
 
 
 @register_test()
 class sessionrestore_no_auto_restore(sessionrestore):
     """
     A start up test measuring the time it takes to load a sessionstore.js file.
 
     1. Set up Firefox to *not* restore automatically from sessionstore.js file.
@@ -184,31 +187,33 @@ class tpaint(TsBase):
     (Measures ctrl-n performance.)
     """
     url = 'file://${talos}/startup_test/tpaint.html?auto=1'
     timeout = 300
     sps_profile_interval = 1
     sps_profile_entries = 2000000
     tpmozafterpaint = True
     filters = filter.ignore_first.prepare(5) + filter.median.prepare()
+    unit = 'ms'
 
 
 @register_test()
 class tresize(TsBase):
     """
     This test does some resize thing.
     """
     extensions = '${talos}/startup_test/tresize/addon'
     cycles = 20
     url = 'startup_test/tresize/addon/content/tresize-test.html'
     timeout = 150
     sps_profile_interval = 2
     sps_profile_entries = 1000000
     tpmozafterpaint = True
     filters = filter.ignore_first.prepare(5) + filter.median.prepare()
+    unit = 'ms'
 
 
 # Media Test
 @register_test()
 class media_tests(TsBase):
     """
     Media Performance Tests
     """
@@ -235,17 +240,18 @@ class PageloaderTest(Test):
     keys = ['tpmanifest', 'tpcycles', 'tppagecycles', 'tprender', 'tpchrome',
             'tpmozafterpaint', 'tploadnocache', 'rss', 'mainthread',
             'resolution', 'cycles', 'sps_profile', 'sps_profile_interval',
             'sps_profile_entries', 'tptimeout', 'win_counters', 'w7_counters',
             'linux_counters', 'mac_counters', 'tpscrolltest', 'xperf_counters',
             'timeout', 'shutdown', 'responsiveness', 'profile_path',
             'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
             'filters', 'preferences', 'extensions', 'setup', 'cleanup',
-            'test_name_extension']
+            'test_name_extension', 'lower_is_better', 'unit']
+    unit = 'ms'
 
 
 @register_test()
 class tps(PageloaderTest):
     """
     Tests the amount of time it takes to switch between tabs
     """
     extensions = '${talos}/tests/tabswitch'
@@ -381,16 +387,17 @@ class glterrain(PageloaderTest):
     sps_profile_interval = 10
     sps_profile_entries = 2000000
     win_counters = w7_counters = linux_counters = mac_counters = None
     """ ASAP mode """
     preferences = {'layout.frame_rate': 0,
                    'docshell.event_starvation_delay_hint': 1,
                    'dom.send_after_paint_to_content': False}
     filters = filter.ignore_first.prepare(1) + filter.median.prepare()
+    unit = 'frame interval'
 
 
 @register_test()
 class tp5n(PageloaderTest):
     """
     Tests the time it takes Firefox to load the tp5 web page test set.
 
     The tp5 is an updated web page test set to 100 pages from April 8th, 2011.
@@ -470,16 +477,17 @@ class tp5o_scroll(PageloaderTest):
 
     tpscrolltest = True
     """ASAP mode"""
     tpmozafterpaint = False
     preferences = {'layout.frame_rate': 0,
                    'docshell.event_starvation_delay_hint': 1,
                    'dom.send_after_paint_to_content': False}
     filters = filter.ignore_first.prepare(1) + filter.median.prepare()
+    unit = '1/FPS'
 
 
 @register_test()
 class v8_7(PageloaderTest):
     """
     This is the V8 (version 7) javascript benchmark taken verbatim and
     slightly modified to fit into our pageloader extension and talos harness.
 
@@ -489,16 +497,18 @@ class v8_7(PageloaderTest):
     tpmanifest = '${talos}/tests/v8_7/v8.manifest'
     sps_profile_interval = 1
     sps_profile_entries = 1000000
     tpcycles = 1
     resolution = 20
     tpmozafterpaint = False
     preferences = {'dom.send_after_paint_to_content': False}
     filters = filter.v8_subtest.prepare()
+    unit = 'score'
+    lower_is_better = False
 
 
 @register_test()
 class kraken(PageloaderTest):
     """
     This is the Kraken javascript benchmark taken verbatim and slightly
     modified to fit into our pageloader extension and talos harness.
     """
@@ -522,21 +532,24 @@ class tcanvasmark(PageloaderTest):
     tpcycles = 5
     tppagecycles = 1
     timeout = 900
     sps_profile_interval = 10
     sps_profile_entries = 2500000
     tpmozafterpaint = False
     preferences = {'dom.send_after_paint_to_content': False}
     filters = filter.ignore_first.prepare(1) + filter.median.prepare()
+    unit = 'score'
+    lower_is_better = False
 
 
 class dromaeo(PageloaderTest):
     """abstract base class for dramaeo tests"""
     filters = filter.dromaeo.prepare()
+    lower_is_better = False
 
 
 @register_test()
 class dromaeo_css(dromaeo):
     """
     Dromaeo suite of tests for JavaScript performance testing.
     See the Dromaeo wiki (https://wiki.mozilla.org/Dromaeo)
     for more information.