Bug 1481815 - generate damp subtest alerts. r=rwood
authorJoel Maher <jmaher@mozilla.com>
Wed, 15 Aug 2018 14:47:27 +0000
changeset 486732 41443a067773651862461210bd73a521fb396aaf
parent 486731 2cc6ec2f7f0e1f945858b493133c6a26c0b4c09f
child 486733 1cee6e0d07e7aaf3a152d2eb15e2114937ce4151
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood
bugs1481815
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1481815 - generate damp subtest alerts. r=rwood allow damp to generate alerts for the subtests Differential Revision: https://phabricator.services.mozilla.com/D3203
testing/talos/talos/output.py
testing/talos/talos/test.py
testing/talos/talos/ttest.py
--- a/testing/talos/talos/output.py
+++ b/testing/talos/talos/output.py
@@ -115,16 +115,22 @@ class Output(object):
                         subtest_index += 1
 
                         if test.test_config.get('lower_is_better') is not None:
                             subtest['lowerIsBetter'] = \
                                 test.test_config['lower_is_better']
                         if test.test_config.get('alert_threshold') is not None:
                             subtest['alertThreshold'] = \
                                 test.test_config['alert_threshold']
+                        if test.test_config.get('subtest_alerts') is not None:
+                            subtest['shouldAlert'] = \
+                                test.test_config['subtest_alerts']
+                        if test.test_config.get('alert_threshold') is not None:
+                            subtest['alertThreshold'] = \
+                                test.test_config['alert_threshold']
                         if test.test_config.get('unit'):
                             subtest['unit'] = test.test_config['unit']
 
                 # if there is more than one subtest, calculate a summary result
                 if len(subtests) > 1:
                     suite['value'] = self.construct_results(
                         vals, testname=test.name())
                 if test.test_config.get('lower_is_better') is not None:
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -32,16 +32,18 @@ class Test(object):
     __test__ = False  # not pytest
 
     cycles = None  # number of cycles
     keys = []
     desktop = True
     filters = filter.ignore_first.prepare(1) + filter.median.prepare()
     lower_is_better = True
     alert_threshold = 2.0
+    perfherder_framework = 'talos'
+    subtest_alerts = False
 
     @classmethod
     def name(cls):
         return cls.__name__
 
     @classmethod
     def description(cls):
         if cls.__doc__ is None:
@@ -240,25 +242,27 @@ class tresize(TsBase):
 
 class PageloaderTest(Test):
     """abstract base class for a Talos Pageloader test"""
     extensions = ['${talos}/pageloader']
     tpmanifest = None  # test manifest
     tpcycles = 1  # number of time to run each page
     cycles = None
     timeout = None
+
     keys = ['tpmanifest', 'tpcycles', 'tppagecycles', 'tprender', 'tpchrome',
             'tpmozafterpaint', 'fnbpaint', 'tphero', 'tploadnocache', 'firstpaint',
             'userready', 'testeventmap', 'base_vs_ref', 'mainthread', 'resolution',
             'cycles', 'gecko_profile', 'gecko_profile_interval', 'gecko_profile_entries',
             'tptimeout', 'win_counters', 'w7_counters', 'linux_counters', 'mac_counters',
             'tpscrolltest', 'xperf_counters', 'timeout', 'responsiveness',
             'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
             'format_pagename', 'filters', 'preferences', 'extensions', 'setup', 'cleanup',
-            'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile']
+            'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile',
+            'subtest_alerts', 'perfherder_framework']
 
 
 class QuantumPageloadTest(PageloaderTest):
     """
     Base class for a Quantum Pageload test
     """
     tpcycles = 1
     tppagecycles = 25
@@ -425,16 +429,18 @@ class damp(PageloaderTest):
     tpmozafterpaint = False
     gecko_profile_interval = 10
     gecko_profile_entries = 1000000
     win_counters = w7_counters = linux_counters = mac_counters = None
     filters = filter.ignore_first.prepare(1) + filter.median.prepare()
     preferences = {'devtools.memory.enabled': True,
                    'addon.test.damp.webserver': '${webserver}'}
     unit = 'ms'
+    subtest_alerts = True
+    perfherder_framework = 'devtools'
 
 
 @register_test()
 class glterrain(PageloaderTest):
     """
     Simple rotating WebGL scene with moving light source over a
     textured terrain.
     Measures average frame intervals.
--- a/testing/talos/talos/ttest.py
+++ b/testing/talos/talos/ttest.py
@@ -141,16 +141,21 @@ class TTest(object):
         for i in range(test_config['cycles']):
             time.sleep(0.25)
             LOG.info('Running cycle %d/%d for %s test...'
                      % (i+1, test_config['cycles'], test_config['name']))
 
             # remove the browser  error file
             mozfile.remove(browser_config['error_filename'])
 
+            # individual tests can have different frameworks
+            # TODO: ensure that we don't run >1 test with custom frameworks
+            if test_config.get('perfherder_framework', None) is not None:
+                test_results.framework = test_config['perfherder_framework']
+
             # reinstall any file whose stability we need to ensure across
             # the cycles
             if test_config.get('reinstall', ''):
                 for keep in test_config['reinstall']:
                     origin = os.path.join(test_config['profile_path'],
                                           keep)
                     dest = os.path.join(setup.profile_dir, keep)
                     LOG.debug('Reinstalling %s on top of %s'