Bug 1222588 - talos should output PERFHERDER_DATA instead of TALOS_DATA. r=jmaher, a=test-only
authorJulien Pagès <j.parkouss@gmail.com>
Sat, 07 Nov 2015 20:35:40 +0100
changeset 305751 2388bca842253f243b3127a8b0f4ae2ada2d4b8e
parent 305750 e83bb92f73cfd6b86e4b56b88027cb3ef79f0c16
child 305752 70abb738e1f82cd1467f52dd3929e182f0a95a08
push id1001
push userraliiev@mozilla.com
push dateMon, 18 Jan 2016 19:06:03 +0000
treeherdermozilla-release@8b89261f3ac4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher, test-only
bugs1222588
milestone44.0a2
Bug 1222588 - talos should output PERFHERDER_DATA instead of TALOS_DATA. r=jmaher, a=test-only
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/talos/output.py
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -36,23 +36,23 @@ TalosErrorList = PythonErrorList + [
      'level': CRITICAL,
      'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
 ]
 
 # TODO: check for running processes on script invocation
 
 class TalosOutputParser(OutputParser):
     minidump_regex = re.compile(r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"''')
-    RE_TALOSDATA = re.compile(r'.*?TALOSDATA:\s+(\[.*\])')
+    RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
     worst_tbpl_status = TBPL_SUCCESS
 
     def __init__(self, **kwargs):
         super(TalosOutputParser, self).__init__(**kwargs)
         self.minidump_output = None
-        self.num_times_found_talosdata = 0
+        self.num_times_found_perf_data = 0
 
     def update_worst_log_and_tbpl_levels(self, log_level, tbpl_level):
         self.worst_log_level = self.worst_level(log_level,
                                                 self.worst_log_level)
         self.worst_tbpl_status = self.worst_level(
             tbpl_level, self.worst_tbpl_status,
             levels=TBPL_WORST_LEVEL_TUPLE
         )
@@ -61,18 +61,18 @@ class TalosOutputParser(OutputParser):
         """ In Talos land, every line that starts with RETURN: needs to be
         printed with a TinderboxPrint:"""
         if line.startswith("RETURN:"):
             line.replace("RETURN:", "TinderboxPrint:")
         m = self.minidump_regex.search(line)
         if m:
             self.minidump_output = (m.group(1), m.group(2), m.group(3))
 
-        if self.RE_TALOSDATA.match(line):
-            self.num_times_found_talosdata += 1
+        if self.RE_PERF_DATA.match(line):
+            self.num_times_found_perf_data += 1
 
         # now let's check if buildbot should retry
         harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
         if harness_retry_re.search(line):
             self.critical(' %s' % line)
             self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY)
             return  # skip base parse_single_line
         super(TalosOutputParser, self).parse_single_line(line)
@@ -363,15 +363,15 @@ class Talos(TestingMixin, MercurialScrip
             if self.return_code == 1:
                 log_level = WARNING
                 tbpl_level = TBPL_WARNING
             if self.return_code == 4:
                 log_level = WARNING
                 tbpl_level = TBPL_RETRY
 
             parser.update_worst_log_and_tbpl_levels(log_level, tbpl_level)
-        elif parser.num_times_found_talosdata != 1:
-            self.critical("TALOSDATA was seen %d times, expected 1."
-                          % parser.num_times_found_talosdata)
+        elif parser.num_times_found_perf_data != 1:
+            self.critical("PERFHERDER_DATA was seen %d times, expected 1."
+                          % parser.num_times_found_perf_data)
             parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING)
 
         self.buildbot_status(parser.worst_tbpl_status,
                              level=parser.worst_log_level)
--- a/testing/talos/talos/output.py
+++ b/testing/talos/talos/output.py
@@ -1,17 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """output formats for Talos"""
 
 import filter
 import json
-import mozinfo
 import logging
 import post_file
 import time
 import utils
 
 # NOTE: we have a circular dependecy with output.py when we import results
 import results as TalosResults
 
@@ -399,198 +398,132 @@ class PerfherderOutput(Output):
         """
 
         # parse the results url
         results_url_split = utils.urlsplit(results_url)
         results_scheme, results_server, results_path, _, _ = results_url_split
 
         # This is the output that treeherder expects to find when parsing the
         # log file
-        logging.info("TALOSDATA: %s" % json.dumps(results))
+        logging.info("PERFHERDER_DATA: %s" % json.dumps(results))
         if results_scheme in ('file'):
             json.dump(results, file(results_path, 'w'), indent=2,
                       sort_keys=True)
 
     def post(self, results, server, path, scheme, tbpl_output):
         """conform to current code- not needed for perfherder"""
         pass
 
-    # TODO: this is copied directly from the old datazilla output. Using it
-    # as we have established platform names already
-    def test_machine(self):
-        """return test machine platform in a form appropriate to datazilla"""
-        platform = mozinfo.os
-        version = mozinfo.version
-        processor = mozinfo.processor
-        if self.results.title.endswith(".e") and \
-                not version.endswith('.e'):
-            # we are running this against e10s builds
-            version = '%s.e' % (version,)
-
-        return dict(name=self.results.title, os=platform, osversion=version,
-                    platform=processor)
-
-    # TODO: this is copied from datazilla output code, do we need all of this?
-    def run_options(self, test):
-        """test options for datazilla"""
-
-        options = {}
-        test_options = ['rss', 'cycles', 'tpmozafterpaint', 'responsiveness',
-                        'shutdown']
-        if 'tpmanifest' in test.test_config:
-            test_options += ['tpchrome', 'tpcycles', 'tppagecycles',
-                             'tprender', 'tploadaboutblank', 'tpdelay']
-
-        for option in test_options:
-            if option not in test.test_config:
-                continue
-            options[option] = test.test_config[option]
-        if test.extensions is not None:
-            options['extensions'] = [{'name': extension}
-                                     for extension in test.extensions]
-        return options
-
     def construct_results(self, vals, testname):
         if 'responsiveness' in testname:
             return self.responsiveness_Metric([val for (val, page) in vals])
         elif testname.startswith('v8_7'):
             return self.v8_Metric(vals)
         elif testname.startswith('kraken'):
             return self.JS_Metric(vals)
         elif testname.startswith('tcanvasmark'):
             return self.CanvasMark_Metric(vals)
         elif len(vals) > 1:
             return filter.geometric_mean([i for i, j in vals])
         else:
             return filter.mean([i for i, j in vals])
 
     def __call__(self):
-        # platform
-        machine = self.test_machine()
-
-        # build information
-        browser_config = self.results.browser_config
-
-        test_results = []
+        suites = []
+        test_results = {
+            'framework': {
+                'name': 'talos',
+            },
+            'suites': suites,
+        }
 
         for test in self.results.results:
-            test_result = {
-                'test_machine': {},
-                'testrun': {},
-                'results': {},
-                'talos_counters': {},
-                'test_build': {}
-            }
-
-            test_result['testrun']['suite'] = test.name()
-            test_result['testrun']['options'] = self.run_options(test)
-            test_result['testrun']['date'] = self.results.date
-
             # serialize test results
-            results = {}
             tsresult = None
-            summary = {"suite": 0, "subtests": {}}
             if not test.using_xperf:
+                subtests = []
+                suite = {
+                    'name': test.name(),
+                    'subtests': subtests,
+                }
+                suites.append(suite)
                 vals = []
+                replicates = {}
 
                 # TODO: counters!!!! we don't have any, but they suffer the same
                 for result in test.results:
                     # XXX this will not work for manifests which list
                     # the same page name twice. It also ignores cycles
                     for page, val in result.raw_values():
                         if page == 'NULL':
-                            results.setdefault(test.name(), []).extend(val)
+                            page = test.name()
                             if tsresult is None:
                                 tsresult = r = TalosResults.Results()
                                 r.results = [{'index': 0, 'page': test.name(),
                                               'runs': val}]
                             else:
                                 r = tsresult.results[0]
                                 if r['page'] == test.name():
                                     r['runs'].extend(val)
-                        else:
-                            results.setdefault(page, []).extend(val)
+                        replicates.setdefault(page, []).extend(val)
 
                 tresults = [tsresult] if tsresult else test.results
 
                 for result in tresults:
                     filtered_results = \
-                        result.values(test_result['testrun']['suite'],
+                        result.values(suite['name'],
                                       test.test_config['filters'])
                     vals.extend([[i['value'], j] for i, j in filtered_results])
                     for val, page in filtered_results:
-                        measurement_metadata = {}
+                        if page == 'NULL':
+                            # no real subtests
+                            page = test.name()
+                        subtest = {
+                            'name': page,
+                            'value': val['filtered'],
+                            'replicates': replicates[page],
+                        }
+                        subtests.append(subtest)
                         if test.test_config.get('lower_is_better') is not None:
-                            measurement_metadata['lowerIsBetter'] = test.test_config['lower_is_better']
+                            subtest['lowerIsBetter'] = test.test_config['lower_is_better']
                         if test.test_config.get('unit'):
-                            measurement_metadata['unit'] = test.test_config['unit']
-                        if page == 'NULL':
-                            summary['subtests'][test.name()] = val
-                            summary['subtests'][test.name()].update(measurement_metadata)
-                        else:
-                            summary['subtests'][page] = val
-                            summary['subtests'][page].update(measurement_metadata)
+                            subtest['unit'] = test.test_config['unit']
 
-                suite_summary = self.construct_results(vals,
-                                                       testname=test.name())
-                summary['suite'] = suite_summary
+                suite['value'] = self.construct_results(vals,
+                                                        testname=test.name())
                 if test.test_config.get('lower_is_better') is not None:
-                    summary['lowerIsBetter'] = test.test_config['lower_is_better']
-                test_result['summary'] = summary
-
-                for result, values in results.items():
-                    test_result['results'][result] = values
+                    suite['lowerIsBetter'] = test.test_config['lower_is_better']
 
             # counters results_aux data
+            counter_subtests = []
             for cd in test.all_counter_results:
                 for name, vals in cd.items():
                     # We want to add the xperf data as talos_counters
                     # exclude counters whose values are tuples (bad for
                     # graphserver)
                     if len(vals) > 0 and isinstance(vals[0], list):
                         continue
 
                     # mainthread IO is a list of filenames and accesses, we do
                     # not report this as a counter
                     if 'mainthreadio' in name:
                         continue
 
-                    if test.using_xperf:
-                        test_result['talos_counters'][name] = {"mean": vals[0]}
-                    else:
-                        # calculate mean and max value
-                        varray = []
-                        counter_mean = 0
-                        counter_max = 0
-                        if len(vals) > 0:
-                            for v in vals:
-                                varray.append(float(v))
-                            counter_mean = "%.2f" % filter.mean(varray)
-                            counter_max = "%.2f" % max(varray)
-                        test_result['talos_counters'][name] = {
-                            "mean": counter_mean,
-                            "max": counter_max
-                        }
+                    subtest = {
+                        'name': name,
+                        'value': 0.0,
+                    }
+                    counter_subtests.append(subtest)
 
-            if browser_config['develop'] and not browser_config['sourcestamp']:
-                browser_config['sourcestamp'] = ''
-
-            test_result['test_build'] = {
-                'version': browser_config['browser_version'],
-                'revision': browser_config['sourcestamp'],
-                'id': browser_config['buildid'],
-                'branch': browser_config['branch_name'],
-                'name': browser_config['browser_name']
-            }
-
-            test_result['test_machine'] = {
-                'platform': machine['platform'],
-                'osversion': machine['osversion'],
-                'os': machine['os'],
-                'name': machine['name']
-            }
-
-            test_results.append(test_result)
+                    if test.using_xperf:
+                        subtest['value'] = vals[0]
+                    else:
+                        # calculate mean value
+                        if len(vals) > 0:
+                            varray = [float(v) for v in vals]
+                            subtest['value'] = "%.2f" % filter.mean(varray)
+            if counter_subtests:
+                suites.append({'name': test.name(), 'subtests': counter_subtests})
         return test_results
 
 # available output formats
 formats = {'datazilla_urls': PerfherderOutput,
            'results_urls': GraphserverOutput}