Bug 1222588 - talos should output PERFHERDER_DATA instead of TALOS_DATA. r=jmaher
authorJulien Pagès <j.parkouss@gmail.com>
Sat, 07 Nov 2015 20:35:40 +0100
changeset 273014 fe34d01fb2ecb2dd4cda82e788cf7b541d5cbdb4
parent 273013 a43c962a140f1f83fe7ceba5ff4e31346ecc7e82
child 273015 db0f569d52a336e9d6898c2c357303d8b34c938f
push id29693
push usercbook@mozilla.com
push dateWed, 18 Nov 2015 13:50:33 +0000
treeherdermozilla-central@1d6155d7e6c9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1222588
milestone45.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1222588 - talos should output PERFHERDER_DATA instead of TALOS_DATA. r=jmaher
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/talos/output.py
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -36,23 +36,23 @@ TalosErrorList = PythonErrorList + [
      'level': CRITICAL,
      'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
 ]
 
 # TODO: check for running processes on script invocation
 
 class TalosOutputParser(OutputParser):
     minidump_regex = re.compile(r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"''')
-    RE_TALOSDATA = re.compile(r'.*?TALOSDATA:\s+(\[.*\])')
+    RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
     worst_tbpl_status = TBPL_SUCCESS
 
     def __init__(self, **kwargs):
         super(TalosOutputParser, self).__init__(**kwargs)
         self.minidump_output = None
-        self.num_times_found_talosdata = 0
+        self.num_times_found_perf_data = 0
 
     def update_worst_log_and_tbpl_levels(self, log_level, tbpl_level):
         self.worst_log_level = self.worst_level(log_level,
                                                 self.worst_log_level)
         self.worst_tbpl_status = self.worst_level(
             tbpl_level, self.worst_tbpl_status,
             levels=TBPL_WORST_LEVEL_TUPLE
         )
@@ -61,18 +61,18 @@ class TalosOutputParser(OutputParser):
         """ In Talos land, every line that starts with RETURN: needs to be
         printed with a TinderboxPrint:"""
         if line.startswith("RETURN:"):
             line.replace("RETURN:", "TinderboxPrint:")
         m = self.minidump_regex.search(line)
         if m:
             self.minidump_output = (m.group(1), m.group(2), m.group(3))
 
-        if self.RE_TALOSDATA.match(line):
-            self.num_times_found_talosdata += 1
+        if self.RE_PERF_DATA.match(line):
+            self.num_times_found_perf_data += 1
 
         # now let's check if buildbot should retry
         harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
         if harness_retry_re.search(line):
             self.critical(' %s' % line)
             self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY)
             return  # skip base parse_single_line
         super(TalosOutputParser, self).parse_single_line(line)
@@ -363,15 +363,15 @@ class Talos(TestingMixin, MercurialScrip
             if self.return_code == 1:
                 log_level = WARNING
                 tbpl_level = TBPL_WARNING
             if self.return_code == 4:
                 log_level = WARNING
                 tbpl_level = TBPL_RETRY
 
             parser.update_worst_log_and_tbpl_levels(log_level, tbpl_level)
-        elif parser.num_times_found_talosdata != 1:
-            self.critical("TALOSDATA was seen %d times, expected 1."
-                          % parser.num_times_found_talosdata)
+        elif parser.num_times_found_perf_data != 1:
+            self.critical("PERFHERDER_DATA was seen %d times, expected 1."
+                          % parser.num_times_found_perf_data)
             parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING)
 
         self.buildbot_status(parser.worst_tbpl_status,
                              level=parser.worst_log_level)
--- a/testing/talos/talos/output.py
+++ b/testing/talos/talos/output.py
@@ -1,17 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """output formats for Talos"""
 
 import filter
 import json
-import mozinfo
 import logging
 import post_file
 import time
 import utils
 
 # NOTE: we have a circular dependecy with output.py when we import results
 import results as TalosResults
 
@@ -399,198 +398,132 @@ class PerfherderOutput(Output):
         """
 
         # parse the results url
         results_url_split = utils.urlsplit(results_url)
         results_scheme, results_server, results_path, _, _ = results_url_split
 
         # This is the output that treeherder expects to find when parsing the
         # log file
-        logging.info("TALOSDATA: %s" % json.dumps(results))
+        logging.info("PERFHERDER_DATA: %s" % json.dumps(results))
         if results_scheme in ('file'):
             json.dump(results, file(results_path, 'w'), indent=2,
                       sort_keys=True)
 
     def post(self, results, server, path, scheme, tbpl_output):
         """conform to current code- not needed for perfherder"""
         pass
 
-    # TODO: this is copied directly from the old datazilla output. Using it
-    # as we have established platform names already
-    def test_machine(self):
-        """return test machine platform in a form appropriate to datazilla"""
-        platform = mozinfo.os
-        version = mozinfo.version
-        processor = mozinfo.processor
-        if self.results.title.endswith(".e") and \
-                not version.endswith('.e'):
-            # we are running this against e10s builds
-            version = '%s.e' % (version,)
-
-        return dict(name=self.results.title, os=platform, osversion=version,
-                    platform=processor)
-
-    # TODO: this is copied from datazilla output code, do we need all of this?
-    def run_options(self, test):
-        """test options for datazilla"""
-
-        options = {}
-        test_options = ['rss', 'cycles', 'tpmozafterpaint', 'responsiveness',
-                        'shutdown']
-        if 'tpmanifest' in test.test_config:
-            test_options += ['tpchrome', 'tpcycles', 'tppagecycles',
-                             'tprender', 'tploadaboutblank', 'tpdelay']
-
-        for option in test_options:
-            if option not in test.test_config:
-                continue
-            options[option] = test.test_config[option]
-        if test.extensions is not None:
-            options['extensions'] = [{'name': extension}
-                                     for extension in test.extensions]
-        return options
-
     def construct_results(self, vals, testname):
         if 'responsiveness' in testname:
             return self.responsiveness_Metric([val for (val, page) in vals])
         elif testname.startswith('v8_7'):
             return self.v8_Metric(vals)
         elif testname.startswith('kraken'):
             return self.JS_Metric(vals)
         elif testname.startswith('tcanvasmark'):
             return self.CanvasMark_Metric(vals)
         elif len(vals) > 1:
             return filter.geometric_mean([i for i, j in vals])
         else:
             return filter.mean([i for i, j in vals])
 
     def __call__(self):
-        # platform
-        machine = self.test_machine()
-
-        # build information
-        browser_config = self.results.browser_config
-
-        test_results = []
+        suites = []
+        test_results = {
+            'framework': {
+                'name': 'talos',
+            },
+            'suites': suites,
+        }
 
         for test in self.results.results:
-            test_result = {
-                'test_machine': {},
-                'testrun': {},
-                'results': {},
-                'talos_counters': {},
-                'test_build': {}
-            }
-
-            test_result['testrun']['suite'] = test.name()
-            test_result['testrun']['options'] = self.run_options(test)
-            test_result['testrun']['date'] = self.results.date
-
             # serialize test results
-            results = {}
             tsresult = None
-            summary = {"suite": 0, "subtests": {}}
             if not test.using_xperf:
+                subtests = []
+                suite = {
+                    'name': test.name(),
+                    'subtests': subtests,
+                }
+                suites.append(suite)
                 vals = []
+                replicates = {}
 
                 # TODO: counters!!!! we don't have any, but they suffer the same
                 for result in test.results:
                     # XXX this will not work for manifests which list
                     # the same page name twice. It also ignores cycles
                     for page, val in result.raw_values():
                         if page == 'NULL':
-                            results.setdefault(test.name(), []).extend(val)
+                            page = test.name()
                             if tsresult is None:
                                 tsresult = r = TalosResults.Results()
                                 r.results = [{'index': 0, 'page': test.name(),
                                               'runs': val}]
                             else:
                                 r = tsresult.results[0]
                                 if r['page'] == test.name():
                                     r['runs'].extend(val)
-                        else:
-                            results.setdefault(page, []).extend(val)
+                        replicates.setdefault(page, []).extend(val)
 
                 tresults = [tsresult] if tsresult else test.results
 
                 for result in tresults:
                     filtered_results = \
-                        result.values(test_result['testrun']['suite'],
+                        result.values(suite['name'],
                                       test.test_config['filters'])
                     vals.extend([[i['value'], j] for i, j in filtered_results])
                     for val, page in filtered_results:
-                        measurement_metadata = {}
+                        if page == 'NULL':
+                            # no real subtests
+                            page = test.name()
+                        subtest = {
+                            'name': page,
+                            'value': val['filtered'],
+                            'replicates': replicates[page],
+                        }
+                        subtests.append(subtest)
                         if test.test_config.get('lower_is_better') is not None:
-                            measurement_metadata['lowerIsBetter'] = test.test_config['lower_is_better']
+                            subtest['lowerIsBetter'] = test.test_config['lower_is_better']
                         if test.test_config.get('unit'):
-                            measurement_metadata['unit'] = test.test_config['unit']
-                        if page == 'NULL':
-                            summary['subtests'][test.name()] = val
-                            summary['subtests'][test.name()].update(measurement_metadata)
-                        else:
-                            summary['subtests'][page] = val
-                            summary['subtests'][page].update(measurement_metadata)
+                            subtest['unit'] = test.test_config['unit']
 
-                suite_summary = self.construct_results(vals,
-                                                       testname=test.name())
-                summary['suite'] = suite_summary
+                suite['value'] = self.construct_results(vals,
+                                                        testname=test.name())
                 if test.test_config.get('lower_is_better') is not None:
-                    summary['lowerIsBetter'] = test.test_config['lower_is_better']
-                test_result['summary'] = summary
-
-                for result, values in results.items():
-                    test_result['results'][result] = values
+                    suite['lowerIsBetter'] = test.test_config['lower_is_better']
 
             # counters results_aux data
+            counter_subtests = []
             for cd in test.all_counter_results:
                 for name, vals in cd.items():
                     # We want to add the xperf data as talos_counters
                     # exclude counters whose values are tuples (bad for
                     # graphserver)
                     if len(vals) > 0 and isinstance(vals[0], list):
                         continue
 
                     # mainthread IO is a list of filenames and accesses, we do
                     # not report this as a counter
                     if 'mainthreadio' in name:
                         continue
 
-                    if test.using_xperf:
-                        test_result['talos_counters'][name] = {"mean": vals[0]}
-                    else:
-                        # calculate mean and max value
-                        varray = []
-                        counter_mean = 0
-                        counter_max = 0
-                        if len(vals) > 0:
-                            for v in vals:
-                                varray.append(float(v))
-                            counter_mean = "%.2f" % filter.mean(varray)
-                            counter_max = "%.2f" % max(varray)
-                        test_result['talos_counters'][name] = {
-                            "mean": counter_mean,
-                            "max": counter_max
-                        }
+                    subtest = {
+                        'name': name,
+                        'value': 0.0,
+                    }
+                    counter_subtests.append(subtest)
 
-            if browser_config['develop'] and not browser_config['sourcestamp']:
-                browser_config['sourcestamp'] = ''
-
-            test_result['test_build'] = {
-                'version': browser_config['browser_version'],
-                'revision': browser_config['sourcestamp'],
-                'id': browser_config['buildid'],
-                'branch': browser_config['branch_name'],
-                'name': browser_config['browser_name']
-            }
-
-            test_result['test_machine'] = {
-                'platform': machine['platform'],
-                'osversion': machine['osversion'],
-                'os': machine['os'],
-                'name': machine['name']
-            }
-
-            test_results.append(test_result)
+                    if test.using_xperf:
+                        subtest['value'] = vals[0]
+                    else:
+                        # calculate mean value
+                        if len(vals) > 0:
+                            varray = [float(v) for v in vals]
+                            subtest['value'] = "%.2f" % filter.mean(varray)
+            if counter_subtests:
+                suites.append({'name': test.name(), 'subtests': counter_subtests})
         return test_results
 
 # available output formats
 formats = {'datazilla_urls': PerfherderOutput,
            'results_urls': GraphserverOutput}