Bug 1530869 - Add support for setting per-suite alert thresholds. r=igoldan
authorEric Rahm <erahm@mozilla.com>
Wed, 27 Feb 2019 14:24:48 +0000
changeset 519362 85c4d0c94938bca534fce92b1ed55a658fdb98dd
parent 519361 d209ceb7588c9060ac427ffc76abacce9cdffb02
child 519363 19b29b2e2f5e651518b4ca8e111c3c1dcf61b8c7
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersigoldan
bugs1530869
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1530869 - Add support for setting per-suite alert thresholds. r=igoldan This adds support for specifying per-suite perfherder alert thresholds and updates the base content JS measurement to specify a 0.25% threshold. Differential Revision: https://phabricator.services.mozilla.com/D21292
testing/awsy/awsy/process_perf_data.py
testing/awsy/awsy/test_base_memory_usage.py
--- a/testing/awsy/awsy/process_perf_data.py
+++ b/testing/awsy/awsy/process_perf_data.py
@@ -66,34 +66,38 @@ def update_checkpoint_paths(checkpoint_f
         if paths:
             indices = [i for i, x in enumerate(checkpoints) if name in x['path']]
             if indices:
                 checkpoints[indices[0]]['path'] = paths[idx]
             else:
                 print "found files but couldn't find %s" % name
 
 
-def create_suite(name, node, data_path, checkpoints=CHECKPOINTS):
+def create_suite(name, node, data_path, checkpoints=CHECKPOINTS, alertThreshold=None):
     """
     Creates a suite suitable for adding to a perfherder blob. Calculates the
     geometric mean of the checkpoint values and adds that to the suite as
     well.
 
     :param name: The name of the suite.
     :param node: The path of the data node to extract data from.
     :param data_path: The directory to retrieve data from.
     :param checkpoints: Which checkpoints to include.
+    :param alertThreshold: The percentage of change that triggers an alert.
     """
     suite = {
         'name': name,
         'subtests': [],
         'lowerIsBetter': True,
         'units': 'bytes'
     }
 
+    if alertThreshold:
+        suite['alertThreshold'] = alertThreshold
+
     extra_opts = []
     # The stylo attributes override each other.
     if 'STYLO_FORCE_ENABLED' in os.environ and os.environ['STYLO_FORCE_ENABLED']:
         extra_opts = ["stylo"]
     if 'STYLO_THREADS' in os.environ and os.environ['STYLO_THREADS'] == '1':
         extra_opts = ["stylo-sequential"]
 
     if 'DMD' in os.environ and os.environ['DMD']:
@@ -156,17 +160,18 @@ def create_perf_data(data_path, perf_sui
 
     perf_blob = {
         'framework': {'name': 'awsy'},
         'suites': []
     }
 
     for suite in perf_suites:
         perf_blob['suites'].append(create_suite(
-            suite['name'], suite['node'], data_path, checkpoints))
+            suite['name'], suite['node'], data_path, checkpoints,
+            suite.get('alertThreshold')))
 
     return perf_blob
 
 
 if __name__ == '__main__':
     args = sys.argv[1:]
     if not args:
         print "Usage: process_perf_data.py data_path"
--- a/testing/awsy/awsy/test_base_memory_usage.py
+++ b/testing/awsy/awsy/test_base_memory_usage.py
@@ -20,17 +20,17 @@ CHECKPOINTS = [
         'median': True,  # We want the median from all content processes
     },
 ]
 
 # A description of each perfherder suite and the path to its values.
 PERF_SUITES = [
     {'name': "Base Content Resident Unique Memory", 'node': "resident-unique"},
     {'name': "Base Content Heap Unclassified", 'node': "explicit/heap-unclassified"},
-    {'name': "Base Content JS", 'node': "js-main-runtime/"},
+    {'name': "Base Content JS", 'node': "js-main-runtime/", 'alertThreshold': 0.25},
     {'name': "Base Content Explicit", 'node': "explicit/"},
 ]
 
 
 class TestMemoryUsage(AwsyTestCase):
     """
     Provides a base case test that just loads about:memory and reports the
     memory usage of a single content process.