Bug 1548845 - [raptor] Allow tests to specify the alertChangeType. r=perftest-reviewers,rwood
☠☠ backed out by f72947acdfcd ☠ ☠
authorHenrik Skupin <mail@hskupin.info>
Fri, 17 May 2019 11:12:20 +0000
changeset 536214 152615db9db653c273b5e90487d198ae2fd788a5
parent 536213 6b3a8394727fbde512c4ee39a4d8d21a17f10581
child 536215 a2544ca8c593cd50ac0753fdc2f799f39c0b057e
push id2082
push userffxbld-merge
push dateMon, 01 Jul 2019 08:34:18 +0000
treeherdermozilla-release@2fb19d0466d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, rwood
bugs1548845
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1548845 - [raptor] Allow tests to specify the alertChangeType. r=perftest-reviewers,rwood Differential Revision: https://phabricator.services.mozilla.com/D30531
testing/raptor/raptor/manifest.py
testing/raptor/raptor/output.py
testing/raptor/test/test_manifest.py
testing/raptor/webext/raptor/runner.js
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -161,16 +161,19 @@ def write_test_settings_json(args, test_
     if subtest_lower_is_better is None:
         # default to main test values if not set
         test_settings['raptor-options']['subtest_lower_is_better'] = (
             test_settings['raptor-options']['lower_is_better'])
     else:
         test_settings['raptor-options']['subtest_lower_is_better'] = bool_from_str(
             subtest_lower_is_better)
 
+    if test_details.get("alert_change_type", None) is not None:
+        test_settings['raptor-options']['alert_change_type'] = test_details['alert_change_type']
+
     if test_details.get("alert_threshold", None) is not None:
         test_settings['raptor-options']['alert_threshold'] = float(test_details['alert_threshold'])
 
     if test_details.get("screen_capture", None) is not None:
         test_settings['raptor-options']['screen_capture'] = test_details.get("screen_capture")
 
     # if Gecko profiling is enabled, write profiling settings for webext
     if test_details.get("gecko_profile", False):
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -55,16 +55,20 @@ class Output(object):
                 'type': test.type,
                 'extraOptions': test.extra_options,
                 'subtests': subtests,
                 'lowerIsBetter': test.lower_is_better,
                 'unit': test.unit,
                 'alertThreshold': float(test.alert_threshold)
             }
 
+            # Check if optional properties have been set by the test
+            if hasattr(test, "alert_change_type"):
+                suite['alertChangeType'] = test.alert_change_type
+
             # if cold load add that info to the suite result dict; this will be used later
             # when combining the results from multiple browser cycles into one overall result
             if test.cold is True:
                 suite['cold'] = True
                 suite['browser_cycle'] = int(test.browser_cycle)
                 suite['expected_browser_cycles'] = int(test.expected_browser_cycles)
 
             suites.append(suite)
--- a/testing/raptor/test/test_manifest.py
+++ b/testing/raptor/test/test_manifest.py
@@ -46,16 +46,17 @@ VALID_MANIFESTS = [{
     'lower_is_better': True,
     'manifest': 'valid_details_1',
     'measure': 'fnbpaint, fcb',
     'page_cycles': 25,
     'test_url': 'http://www.test-url/goes/here',
     'type': 'pageload',
     'unit': 'ms',
 
+    'alert_change_type': None,
     'alert_on': None,
     'playback': None,
 }, {
     # page load test for geckoview
     'alert_threshold': 2.0,
     'apps': 'geckoview',
     'browser_cycles': 10,
     'cold': True,
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -63,25 +63,28 @@ var isBenchmarkPending = false;
 var pageTimeout = 10000; // default pageload timeout
 var geckoProfiling = false;
 var geckoInterval = 1;
 var geckoEntries = 1000000;
 var geckoThreads = [];
 var debugMode = 0;
 var screenCapture = false;
 
-var results = {"name": "",
-               "page": "",
-               "type": "",
-               "browser_cycle": 0,
-               "expected_browser_cycles": 0,
-               "cold": false,
-               "lower_is_better": true,
-               "alert_threshold": 2.0,
-               "measurements": {}};
+var results = {
+  "name": "",
+  "page": "",
+  "type": "",
+  "browser_cycle": 0,
+  "expected_browser_cycles": 0,
+  "cold": false,
+  "lower_is_better": true,
+  "alert_change_type": "relative",
+  "alert_threshold": 2.0,
+  "measurements": {},
+};
 
 function getTestSettings() {
   console.log("getting test settings from control server");
   return new Promise(resolve => {
     fetch(settingsURL).then(function(response) {
       response.text().then(function(text) {
         console.log(text);
         settings = JSON.parse(text)["raptor-options"];
@@ -103,27 +106,28 @@ function getTestSettings() {
 
         if (host) {
           // just replace the '<host>' keyword in the URL with actual host
           testURL = testURL.replace("<host>", host);
         }
 
         console.log(`testURL: ${testURL}`);
 
+        results.alert_change_type = settings.alert_change_type;
+        results.alert_threshold = settings.alert_threshold;
+        results.browser_cycle = browserCycle;
+        results.cold = settings.cold;
+        results.expected_browser_cycles = settings.expected_browser_cycles;
+        results.lower_is_better = settings.lower_is_better === true;
+        results.name = testName;
         results.page = testURL;
         results.type = testType;
-        results.name = testName;
-        results.browser_cycle = browserCycle;
-        results.expected_browser_cycles = settings.expected_browser_cycles;
-        results.cold = settings.cold;
         results.unit = settings.unit;
         results.subtest_unit = settings.subtest_unit;
-        results.lower_is_better = settings.lower_is_better === true;
         results.subtest_lower_is_better = settings.subtest_lower_is_better === true;
-        results.alert_threshold = settings.alert_threshold;
 
         if (settings.gecko_profile === true) {
           results.extra_options = ["gecko_profile"];
 
           geckoProfiling = true;
           geckoEntries = settings.gecko_profile_entries;
           geckoInterval = settings.gecko_profile_interval;
           geckoThreads = settings.gecko_profile_threads;