Bug 1549033 - Only dump PERFHERDER_DATA when scenario tests are run with a resource usage flag. r=perftest-reviewers,stephendonner,rwood
authorGregory Mierzwinski <gmierz2@outlook.com>
Thu, 04 Jul 2019 15:36:40 +0000
changeset 544145 ac31aae4bcd4ac623bfdc4e0af50a3bbdbb437f4
parent 544144 44acade85bf20f114e3d9a64c446bed5c0124112
child 544146 2f763c73129de66f6230720b28851ed3ec60b621
push id2131
push userffxbld-merge
push dateMon, 26 Aug 2019 18:30:20 +0000
treeherdermozilla-release@b19ffb3ca153 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, stephendonner, rwood
bugs1549033
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1549033 - Only dump PERFHERDER_DATA when scenario tests are run with a resource usage flag. r=perftest-reviewers,stephendonner,rwood For the Raptor 'scenario' test type, this patch prevents PERFHERDER_DATA from being output when `--power-test`, `--cpu-test`, or `--memory-test` are not used. Differential Revision: https://phabricator.services.mozilla.com/D31665
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/raptor/raptor/output.py
testing/raptor/raptor/raptor.py
testing/raptor/raptor/results.py
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -1,17 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import copy
-import json
 import os
 import re
 import sys
 import subprocess
 
 from shutil import copyfile
 
 import mozharness
@@ -517,44 +516,16 @@ class Raptor(TestingMixin, MercurialScri
         if self.gecko_profile and self.run_local:
             tools = os.path.join(self.config['repo_path'], 'testing', 'tools')
             view_gecko_profile_req = os.path.join(tools,
                                                   'view_gecko_profile',
                                                   'requirements.txt')
             self.info("installing requirements for the view-gecko-profile tool")
             self.install_module(requirements=[view_gecko_profile_req])
 
-    def _validate_treeherder_data(self, parser):
-        # late import is required, because install is done in create_virtualenv
-        import jsonschema
-
-        expected_perfherder = 1
-        if self.config.get('power_test', None):
-            expected_perfherder += 1
-        if self.config.get('memory_test', None):
-            expected_perfherder += 1
-        if self.config.get('cpu_test', None):
-            expected_perfherder += 1
-        if len(parser.found_perf_data) != expected_perfherder:
-            self.critical("PERFHERDER_DATA was seen %d times, expected %d."
-                          % (len(parser.found_perf_data), expected_perfherder))
-            return
-
-        schema_path = os.path.join(external_tools_path,
-                                   'performance-artifact-schema.json')
-        self.info("Validating PERFHERDER_DATA against %s" % schema_path)
-        try:
-            with open(schema_path) as f:
-                schema = json.load(f)
-            data = json.loads(parser.found_perf_data[0])
-            jsonschema.validate(data, schema)
-        except Exception as e:
-            self.exception("Error while validating PERFHERDER_DATA")
-            self.info(str(e))
-
     def _artifact_perf_data(self, src, dest):
         if not os.path.isdir(os.path.dirname(dest)):
             # create upload dir if it doesn't already exist
             self.info("creating dir: %s" % os.path.dirname(dest))
             os.makedirs(os.path.dirname(dest))
         self.info('copying raptor results from %s to %s' % (src, dest))
         try:
             copyfile(src, dest)
@@ -641,45 +612,42 @@ class Raptor(TestingMixin, MercurialScri
         if self.app in self.firefox_android_browsers:
             self.logcat_stop()
 
         if parser.minidump_output:
             self.info("Looking at the minidump files for debugging purposes...")
             for item in parser.minidump_output:
                 self.run_command(["ls", "-l", item])
 
-        elif '--no-upload-results' not in options:
-            if not self.gecko_profile:
-                self._validate_treeherder_data(parser)
-            if not self.run_local:
-                # copy results to upload dir so they are included as an artifact
-                self.info("copying raptor results to upload dir:")
+        elif not self.run_local:
+            # copy results to upload dir so they are included as an artifact
+            self.info("copying raptor results to upload dir:")
 
-                src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor.json')
-                dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
-                self.info(str(dest))
+            src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor.json')
+            dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
+            self.info(str(dest))
+            self._artifact_perf_data(src, dest)
+
+            if self.power_test:
+                src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-power.json')
                 self._artifact_perf_data(src, dest)
 
-                if self.power_test:
-                    src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-power.json')
-                    self._artifact_perf_data(src, dest)
-
-                if self.memory_test:
-                    src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-memory.json')
-                    self._artifact_perf_data(src, dest)
+            if self.memory_test:
+                src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-memory.json')
+                self._artifact_perf_data(src, dest)
 
-                if self.cpu_test:
-                    src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-cpu.json')
-                    self._artifact_perf_data(src, dest)
+            if self.cpu_test:
+                src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-cpu.json')
+                self._artifact_perf_data(src, dest)
 
-                src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'screenshots.html')
-                if os.path.exists(src):
-                    dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'screenshots.html')
-                    self.info(str(dest))
-                    self._artifact_perf_data(src, dest)
+            src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'screenshots.html')
+            if os.path.exists(src):
+                dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'screenshots.html')
+                self.info(str(dest))
+                self._artifact_perf_data(src, dest)
 
 
 class RaptorOutputParser(OutputParser):
     minidump_regex = re.compile(r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''')
     RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
 
     def __init__(self, **kwargs):
         super(RaptorOutputParser, self).__init__(**kwargs)
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -834,53 +834,67 @@ class Output(object):
             with open(screenshot_path, 'w') as f:
                 for result in self.summarized_screenshots:
                     f.write("%s\n" % result)
             LOG.info("screen captures can be found locally at: %s" % screenshot_path)
 
         # now that we've checked for screen captures too, if there were no actual
         # test results we can bail out here
         if self.summarized_results == {}:
-            return False
+            return False, 0
 
         # when gecko_profiling, we don't want results ingested by Perfherder
         extra_opts = self.summarized_results['suites'][0].get('extraOptions', [])
-        if 'gecko_profile' not in extra_opts:
+        test_type = self.summarized_results['suites'][0].get('type', '')
+
+        output_perf_data = True
+        not_posting = '- not posting regular test results for perfherder'
+        if 'gecko_profile' in extra_opts:
+            LOG.info("gecko profiling enabled %s" % not_posting)
+            output_perf_data = False
+        elif test_type == 'scenario':
+            # if a resource-usage flag was supplied the perfherder data
+            # will still be output from output_supporting_data
+            LOG.info("scenario test type was run %s" % not_posting)
+            output_perf_data = False
+
+        total_perfdata = 0
+        if output_perf_data:
             # if we have supporting data i.e. power, we ONLY want those measurements
             # dumped out. TODO: Bug 1515406 - Add option to output both supplementary
             # data (i.e. power) and the regular Raptor test result
             # Both are already available as separate PERFHERDER_DATA json blobs
             if len(self.summarized_supporting_data) == 0:
                 LOG.info("PERFHERDER_DATA: %s" % json.dumps(self.summarized_results))
+                total_perfdata = 1
             else:
                 LOG.info("supporting data measurements exist - only posting those to perfherder")
-        else:
-            LOG.info("gecko profiling enabled - not posting results for perfherder")
 
         json.dump(self.summarized_results, open(results_path, 'w'), indent=2,
                   sort_keys=True)
         LOG.info("results can also be found locally at: %s" % results_path)
 
-        return True
+        return True, total_perfdata
 
     def output_supporting_data(self, test_names):
         '''
         Supporting data was gathered outside of the main raptor test; it has already
         been summarized, now output it appropriately.
 
         We want to output supporting data in a completely separate perfherder json blob and
         in a corresponding file artifact. This way supporting data can be ingested as it's own
         test suite in perfherder and alerted upon if desired. Kept outside of the test results
         from the actual Raptor test that was ran when the supporting data was gathered.
         '''
         if len(self.summarized_supporting_data) == 0:
             LOG.error("no summarized supporting data found for %s" %
                       ', '.join(test_names))
-            return False
+            return False, 0
 
+        total_perfdata = 0
         for next_data_set in self.summarized_supporting_data:
             data_type = next_data_set['suites'][0]['type']
 
             if os.environ['MOZ_UPLOAD_DIR']:
                 # i.e. testing/mozharness/build/raptor.json locally; in production it will
                 # be at /tasks/task_*/build/ (where it will be picked up by mozharness later
                 # and made into a tc artifact accessible in treeherder as perfherder-data.json)
                 results_path = os.path.join(os.path.dirname(os.environ['MOZ_UPLOAD_DIR']),
@@ -889,18 +903,19 @@ class Output(object):
                 results_path = os.path.join(os.getcwd(), 'raptor-%s.json' % data_type)
 
             # dump data to raptor-data.json artifact
             json.dump(next_data_set, open(results_path, 'w'), indent=2, sort_keys=True)
 
             # the output that treeherder expects to find
             LOG.info("PERFHERDER_DATA: %s" % json.dumps(next_data_set))
             LOG.info("%s results can also be found locally at: %s" % (data_type, results_path))
+            total_perfdata += 1
 
-        return True
+        return True, total_perfdata
 
     @classmethod
     def v8_Metric(cls, val_list):
         results = [i for i, j in val_list]
         score = 100 * filters.geometric_mean(results)
         return score
 
     @classmethod
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -131,17 +131,17 @@ class Raptor(object):
         if self.debug_mode:
             self.post_startup_delay = min(self.post_startup_delay, 3000)
             LOG.info("debug-mode enabled, reducing post-browser startup pause to %d ms"
                      % self.post_startup_delay)
 
         LOG.info("main raptor init, config is: %s" % str(self.config))
 
         # setup the control server
-        self.results_handler = RaptorResultsHandler()
+        self.results_handler = RaptorResultsHandler(self.config)
         self.start_control_server()
 
         self.build_browser_profile()
 
     @property
     def profile_data_dir(self):
         if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
             return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
@@ -180,17 +180,16 @@ class Raptor(object):
         self.config['subtest_alert_on'] = test.get('alert_on')
 
     def run_tests(self, tests, test_names):
         try:
             for test in tests:
                 self.run_test(test, timeout=int(test.get('page_timeout')))
 
             return self.process_results(test_names)
-
         finally:
             self.clean_up()
 
     def run_test(self, test, timeout):
         raise NotImplementedError()
 
     def wait_for_test_finish(self, test, timeout):
         # this is a 'back-stop' i.e. if for some reason Raptor doesn't finish for some
--- a/testing/raptor/raptor/results.py
+++ b/testing/raptor/raptor/results.py
@@ -1,26 +1,30 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # class to process, format, and report raptor test results
 # received from the raptor control server
 from __future__ import absolute_import
 
+import json
+import os
+
 from logger.logger import RaptorLogger
 from output import Output
 
 LOG = RaptorLogger(component='raptor-results-handler')
 
 
 class RaptorResultsHandler():
     """Handle Raptor test results"""
 
-    def __init__(self):
+    def __init__(self, config=None):
+        self.config = config
         self.results = []
         self.page_timeout_list = []
         self.images = []
         self.supporting_data = None
 
     def add(self, new_result_json):
         # add to results
         LOG.info("received results in RaptorResultsHandler.add")
@@ -66,30 +70,102 @@ class RaptorResultsHandler():
                                'proportional': proportional}}
         '''
         LOG.info("RaptorResultsHandler.add_supporting_data received %s data"
                  % supporting_data['type'])
         if self.supporting_data is None:
             self.supporting_data = []
         self.supporting_data.append(supporting_data)
 
+    def _get_expected_perfherder(self, output):
+        expected_perfherder = 1
+
+        def is_resource_test():
+            if self.config.get('power_test', None) or \
+               self.config.get('cpu_test', None) or \
+               self.config.get('memory_test', None):
+                return True
+            return False
+
+        if not is_resource_test() and \
+           (output.summarized_supporting_data or output.summarized_results):
+            data = output.summarized_supporting_data
+            if not data:
+                data = [output.summarized_results]
+
+            for next_data_set in data:
+                data_type = next_data_set['suites'][0]['type']
+                if data_type == 'scenario':
+                    return None
+
+        if self.config.get('power_test', None):
+            expected_perfherder += 1
+        if self.config.get('memory_test', None):
+            expected_perfherder += 1
+        if self.config.get('cpu_test', None):
+            expected_perfherder += 1
+
+        return expected_perfherder
+
+    def _validate_treeherder_data(self, output, output_perfdata):
+        # late import is required, because install is done in create_virtualenv
+        import jsonschema
+
+        expected_perfherder = self._get_expected_perfherder(output)
+        if expected_perfherder is None:
+            LOG.info(
+                "Skipping PERFHERDER_DATA check "
+                "because no perfherder data output is expected"
+            )
+            return True
+        elif output_perfdata != expected_perfherder:
+            LOG.critical("PERFHERDER_DATA was seen %d times, expected %d."
+                         % (output_perfdata, expected_perfherder))
+            return False
+
+        external_tools_path = os.environ['EXTERNALTOOLSPATH']
+        schema_path = os.path.join(external_tools_path,
+                                   'performance-artifact-schema.json')
+        LOG.info("Validating PERFHERDER_DATA against %s" % schema_path)
+        try:
+            with open(schema_path) as f:
+                schema = json.load(f)
+            if output.summarized_results:
+                data = output.summarized_results
+            else:
+                data = output.summarized_supporting_data[0]
+            jsonschema.validate(data, schema)
+        except Exception as e:
+            LOG.exception("Error while validating PERFHERDER_DATA")
+            LOG.info(str(e))
+            return False
+        return True
+
     def summarize_and_output(self, test_config, test_names):
         # summarize the result data, write to file and output PERFHERDER_DATA
         LOG.info("summarizing raptor test results")
         output = Output(self.results, self.supporting_data, test_config['subtest_alert_on'])
         output.summarize(test_names)
         # that has each browser cycle separate; need to check if there were multiple browser
         # cycles, and if so need to combine results from all cycles into one overall result
         output.combine_browser_cycles()
         output.summarize_screenshots(self.images)
         # only dump out supporting data (i.e. power) if actual Raptor test completed
+        out_sup_perfdata = 0
         if self.supporting_data is not None and len(self.results) != 0:
             output.summarize_supporting_data()
-            output.output_supporting_data(test_names)
-        return output.output(test_names)
+            res, out_sup_perfdata = output.output_supporting_data(test_names)
+        res, out_perfdata = output.output(test_names)
+
+        if not self.config['gecko_profile']:
+            # res will remain True if no problems are encountered
+            # during schema validation and perferder_data counting
+            res = self._validate_treeherder_data(output, out_sup_perfdata + out_perfdata)
+
+        return res
 
 
 class RaptorTestResult():
     """Single Raptor test result class"""
 
     def __init__(self, test_result_json):
         self.extra_options = []
         # convert test result json/dict (from control server) to test result object instance