Bug 1548845 - [raptor] Allow tests to specify the alertChangeType. r=perftest-reviewers,rwood
authorHenrik Skupin <mail@hskupin.info>
Fri, 17 May 2019 20:14:25 +0000
changeset 474417 3f21ab6755853fa760af197ce7d392677b85249a
parent 474416 7dc6dbc72a41b56669726ebc835aeb5cf0ded1ab
child 474418 8ac4327262cea2ada01adb01dd5b40d844381895
push id85796
push userrwood@mozilla.com
push dateFri, 17 May 2019 22:42:35 +0000
treeherderautoland@6977ccf2ff65 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, rwood
bugs1548845
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1548845 - [raptor] Allow tests to specify the alertChangeType. r=perftest-reviewers,rwood Differential Revision: https://phabricator.services.mozilla.com/D30531
testing/raptor/raptor/manifest.py
testing/raptor/raptor/output.py
testing/raptor/test/test_manifest.py
testing/raptor/webext/raptor/runner.js
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -161,16 +161,19 @@ def write_test_settings_json(args, test_
     if subtest_lower_is_better is None:
         # default to main test values if not set
         test_settings['raptor-options']['subtest_lower_is_better'] = (
             test_settings['raptor-options']['lower_is_better'])
     else:
         test_settings['raptor-options']['subtest_lower_is_better'] = bool_from_str(
             subtest_lower_is_better)
 
+    if test_details.get("alert_change_type", None) is not None:
+        test_settings['raptor-options']['alert_change_type'] = test_details['alert_change_type']
+
     if test_details.get("alert_threshold", None) is not None:
         test_settings['raptor-options']['alert_threshold'] = float(test_details['alert_threshold'])
 
     if test_details.get("screen_capture", None) is not None:
         test_settings['raptor-options']['screen_capture'] = test_details.get("screen_capture")
 
     # if Gecko profiling is enabled, write profiling settings for webext
     if test_details.get("gecko_profile", False):
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -55,16 +55,20 @@ class Output(object):
                 'type': test.type,
                 'extraOptions': test.extra_options,
                 'subtests': subtests,
                 'lowerIsBetter': test.lower_is_better,
                 'unit': test.unit,
                 'alertThreshold': float(test.alert_threshold)
             }
 
+            # Check if optional properties have been set by the test
+            if hasattr(test, "alert_change_type"):
+                suite['alertChangeType'] = test.alert_change_type
+
             # if cold load add that info to the suite result dict; this will be used later
             # when combining the results from multiple browser cycles into one overall result
             if test.cold is True:
                 suite['cold'] = True
                 suite['browser_cycle'] = int(test.browser_cycle)
                 suite['expected_browser_cycles'] = int(test.expected_browser_cycles)
 
             suites.append(suite)
--- a/testing/raptor/test/test_manifest.py
+++ b/testing/raptor/test/test_manifest.py
@@ -46,16 +46,17 @@ VALID_MANIFESTS = [{
     'lower_is_better': True,
     'manifest': 'valid_details_1',
     'measure': 'fnbpaint, fcb',
     'page_cycles': 25,
     'test_url': 'http://www.test-url/goes/here',
     'type': 'pageload',
     'unit': 'ms',
 
+    'alert_change_type': None,
     'alert_on': None,
     'playback': None,
 }, {
     # page load test for geckoview
     'alert_threshold': 2.0,
     'apps': 'geckoview',
     'browser_cycles': 10,
     'cold': True,
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -63,25 +63,28 @@ var isBenchmarkPending = false;
 var pageTimeout = 10000; // default pageload timeout
 var geckoProfiling = false;
 var geckoInterval = 1;
 var geckoEntries = 1000000;
 var geckoThreads = [];
 var debugMode = 0;
 var screenCapture = false;
 
-var results = {"name": "",
-               "page": "",
-               "type": "",
-               "browser_cycle": 0,
-               "expected_browser_cycles": 0,
-               "cold": false,
-               "lower_is_better": true,
-               "alert_threshold": 2.0,
-               "measurements": {}};
+var results = {
+  "name": "",
+  "page": "",
+  "type": "",
+  "browser_cycle": 0,
+  "expected_browser_cycles": 0,
+  "cold": false,
+  "lower_is_better": true,
+  "alert_change_type": "relative",
+  "alert_threshold": 2.0,
+  "measurements": {},
+};
 
 function getTestSettings() {
   console.log("getting test settings from control server");
   return new Promise(resolve => {
     fetch(settingsURL).then(function(response) {
       response.text().then(function(text) {
         console.log(text);
         settings = JSON.parse(text)["raptor-options"];
@@ -103,27 +106,28 @@ function getTestSettings() {
 
         if (host) {
           // just replace the '<host>' keyword in the URL with actual host
           testURL = testURL.replace("<host>", host);
         }
 
         console.log(`testURL: ${testURL}`);
 
+        results.alert_change_type = settings.alert_change_type;
+        results.alert_threshold = settings.alert_threshold;
+        results.browser_cycle = browserCycle;
+        results.cold = settings.cold;
+        results.expected_browser_cycles = settings.expected_browser_cycles;
+        results.lower_is_better = settings.lower_is_better === true;
+        results.name = testName;
         results.page = testURL;
         results.type = testType;
-        results.name = testName;
-        results.browser_cycle = browserCycle;
-        results.expected_browser_cycles = settings.expected_browser_cycles;
-        results.cold = settings.cold;
         results.unit = settings.unit;
         results.subtest_unit = settings.subtest_unit;
-        results.lower_is_better = settings.lower_is_better === true;
         results.subtest_lower_is_better = settings.subtest_lower_is_better === true;
-        results.alert_threshold = settings.alert_threshold;
 
         if (settings.gecko_profile === true) {
           results.extra_options = ["gecko_profile"];
 
           geckoProfiling = true;
           geckoEntries = settings.gecko_profile_entries;
           geckoInterval = settings.gecko_profile_interval;
           geckoThreads = settings.gecko_profile_threads;