Bug 1193385 - Test summarization for subtests gives incorrect results for tresize (and possibly other tests). r=parkouss
authorJoel Maher <jmaher@mozilla.com>
Mon, 17 Aug 2015 07:50:21 -0400
changeset 996 c0039de7a1c6
parent 995 11b48cfc4512
child 997 306fac2f61aa
push id685
push userjmaher@mozilla.com
push dateMon, 17 Aug 2015 11:50:26 +0000
reviewersparkouss
bugs1193385
Bug 1193385 - Test summarization for subtests gives incorrect results for tresize (and possibly other tests). r=parkouss
talos/filter.py
talos/output.py
talos/test.py
--- a/talos/filter.py
+++ b/talos/filter.py
@@ -180,10 +180,10 @@ def v8_subtest(series, name):
                  'Boyer': 666463.,
                  'NavierStokes': 1484000.,
                  'RayTrace': 739989.,
                  'RegExp': 910985.,
                  'Richards': 35302.,
                  'Splay': 81491.
                  }
 
-    return reference[name] / filter.geometric_mean(series)
+    return reference[name] / geometric_mean(series)
 
--- a/talos/output.py
+++ b/talos/output.py
@@ -7,16 +7,19 @@
 import filter
 import json
 import mozinfo
 import logging
 import post_file
 import time
 import utils
 
+# NOTE: we have a circular dependecy with output.py when we import results
+import results as TalosResults
+
 from StringIO import StringIO
 
 
 def filesizeformat(bytes):
     """
     Format the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB, 102
     bytes, etc).
     """
@@ -484,37 +487,52 @@ class PerfherderOutput(Output):
             }
 
             test_result['testrun']['suite'] = test.name()
             test_result['testrun']['options'] = self.run_options(test)
             test_result['testrun']['date'] = self.results.date
 
             # serialize test results
             results = {}
+            tsresult = None
             summary = {"suite": 0, "subtests": {}}
             if not test.using_xperf:
                 vals = []
+
+                # TODO: counters!!!! we don't have any, but they suffer the same
                 for result in test.results:
+                    # XXX this will not work for manifests which list
+                    # the same page name twice. It also ignores cycles
+                    for page, val in result.raw_values():
+                        if page == 'NULL':
+                            results.setdefault(test.name(), []).extend(val)
+                            if tsresult is None:
+                                tsresult = r = TalosResults.Results()
+                                r.results = [{'index': 0, 'page': test.name(),
+                                              'runs': val}]
+                            else:
+                                r = tsresult.results[0]
+                                if r['page'] == test.name():
+                                    r['runs'].extend(val)
+                        else:
+                            results.setdefault(page, []).extend(val)
+
+                tresults = [tsresult] if tsresult else test.results
+
+                for result in tresults:
                     filtered_results = \
                         result.values(test_result['testrun']['suite'],
                                       test.test_config['filters'])
                     vals.extend([[i['filtered'], j] for i, j in filtered_results])
                     for val, page in filtered_results:
                         if page == 'NULL':
                             summary['subtests'][test.name()] = val
                         else:
                             summary['subtests'][page] = val
 
-                    # XXX this will not work for manifests which list
-                    # the same page name twice. It also ignores cycles
-                    for page, val in result.raw_values():
-                        if page == 'NULL':
-                            results.setdefault(test.name(), []).extend(val)
-                        else:
-                            results.setdefault(page, []).extend(val)
 
                 suite_summary = self.construct_results(vals,
                                                        testname=test.name())
                 summary['suite'] = suite_summary
                 test_result['summary'] = summary
 
                 for result, values in results.items():
                     test_result['results'][result] = values
--- a/talos/test.py
+++ b/talos/test.py
@@ -481,16 +481,17 @@ class v8_7(PageloaderTest):
     """
     tpmanifest = '${talos}/page_load_test/v8_7/v8.manifest'
     sps_profile_interval = 1
     sps_profile_entries = 1000000
     tpcycles = 1
     resolution = 20
     tpmozafterpaint = False
     preferences = {'dom.send_after_paint_to_content': False}
+    filters = filter.v8_subtest.prepare()
 
 
 @register_test()
 class kraken(PageloaderTest):
     """
     This is the Kraken javascript benchmark taken verbatim and slightly
     modified to fit into our pageloader extension and talos harness.
     """