Bug 1504227 - Add raptor test option to specify what measurement(s) to alert on; r=davehunt
authorRob Wood <rwood@mozilla.com>
Thu, 24 Jan 2019 19:06:49 +0000
changeset 515495 9a008f412fd77d52636b6430c07628bfec3b8252
parent 515494 6043695f46a99e8f4c781ffbf1f2858d7c960aba
child 515496 07500fee706b0073f2d36e416e9fdd407968f494
push id1953
push userffxbld-merge
push dateMon, 11 Mar 2019 12:10:20 +0000
treeherdermozilla-release@9c35dcbaa899 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1504227
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1504227 - Add raptor test option to specify what measurement(s) to alert on; r=davehunt Differential Revision: https://phabricator.services.mozilla.com/D17288
testing/raptor/raptor/manifest.py
testing/raptor/raptor/tests/raptor-tp6-1.ini
testing/raptor/raptor/tests/raptor-tp6-10.ini
testing/raptor/raptor/tests/raptor-tp6-2.ini
testing/raptor/raptor/tests/raptor-tp6-3.ini
testing/raptor/raptor/tests/raptor-tp6-4.ini
testing/raptor/raptor/tests/raptor-tp6-5.ini
testing/raptor/raptor/tests/raptor-tp6-6.ini
testing/raptor/raptor/tests/raptor-tp6-7.ini
testing/raptor/raptor/tests/raptor-tp6-8.ini
testing/raptor/raptor/tests/raptor-tp6-9.ini
testing/raptor/raptor/tests/raptor-tp6m-1.ini
testing/raptor/test/test_manifest.py
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -43,27 +43,38 @@ def validate_test_ini(test_details):
     valid_settings = True
 
     for setting in required_settings:
         # measure setting not required for benchmark type tests
         if setting == 'measure' and test_details['type'] == 'benchmark':
             continue
         if setting not in test_details:
             valid_settings = False
-            LOG.info("setting '%s' is required but not found in %s"
+            LOG.error("ERROR: setting '%s' is required but not found in %s"
                      % (setting, test_details['manifest']))
 
     # if playback is specified, we need more playback settings
     if 'playback' in test_details:
         for setting in playback_settings:
             if setting not in test_details:
                 valid_settings = False
-                LOG.info("setting '%s' is required but not found in %s"
+                LOG.error("ERROR: setting '%s' is required but not found in %s"
                          % (setting, test_details['manifest']))
 
+    # if 'alert-on' is specified, we need to make sure that the value given is valid
+    # i.e. any 'alert_on' values must be values that exist in the 'measure' ini setting
+    # i.e. 'alert_on = fcp, loadtime' must get rid of comma and space
+    if 'alert_on' in test_details:
+        for alert_on_value in test_details['alert_on'].split(', '):
+            alert_on_value = alert_on_value.strip()
+            if alert_on_value not in test_details['measure']:
+                LOG.error("ERROR: The 'alert_on' value of '%s' is not valid because " \
+                          "it doesn't exist in the 'measure' test setting!"
+                         % alert_on_value)
+                valid_settings = False
     return valid_settings
 
 
 def write_test_settings_json(args, test_details, oskey):
     # write test settings json file with test details that the control
     # server will provide for the web ext
     test_url = transform_platform(test_details['test_url'], oskey)
 
@@ -80,21 +91,24 @@ def write_test_settings_json(args, test_
         test_settings['raptor-options']['measure'] = {}
         if "dcf" in test_details['measure']:
             test_settings['raptor-options']['measure']['dcf'] = True
         if "fnbpaint" in test_details['measure']:
             test_settings['raptor-options']['measure']['fnbpaint'] = True
         if "fcp" in test_details['measure']:
             test_settings['raptor-options']['measure']['fcp'] = True
         if "hero" in test_details['measure']:
-            test_settings['raptor-options']['measure']['hero'] = test_details['hero'].split()
+            test_settings['raptor-options']['measure']['hero'] = test_details['hero'].split(', ')
         if "ttfi" in test_details['measure']:
             test_settings['raptor-options']['measure']['ttfi'] = True
         if "loadtime" in test_details['measure']:
             test_settings['raptor-options']['measure']['loadtime'] = True
+        if test_details.get("alert_on", None) is not None:
+            # i.e. 'alert_on = fcp, loadtime' must get rid of comma and space
+            test_settings['raptor-options']['alert_on'] = test_details['alert_on'].split(', ')
     if test_details.get("page_timeout", None) is not None:
         test_settings['raptor-options']['page_timeout'] = int(test_details['page_timeout'])
     test_settings['raptor-options']['unit'] = test_details.get("unit", "ms")
     if test_details.get("lower_is_better", "true") == "false":
         test_settings['raptor-options']['lower_is_better'] = False
     else:
         test_settings['raptor-options']['lower_is_better'] = True
 
--- a/testing/raptor/raptor/tests/raptor-tp6-1.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-1.ini
@@ -14,16 +14,17 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
 gecko_profile_interval = 1
 gecko_profile_entries = 14000000
+alert_on = fcp, loadtime
 
 [raptor-tp6-amazon-firefox]
 apps = firefox
 test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
 playback_recordings = amazon.mp
 measure = fnbpaint, fcp, hero, dcf, ttfi, loadtime
 hero = hero1
 
--- a/testing/raptor/raptor/tests/raptor-tp6-10.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-10.ini
@@ -13,16 +13,17 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 60000
 gecko_profile_interval = 1
 gecko_profile_entries = 2000000
+alert_on = fcp, loadtime
 
 [raptor-tp6-paypal-firefox]
 apps = firefox
 test_url = https://www.paypal.com/myaccount/summary/
 playback_pageset_manifest = mitmproxy-recordings-raptor-paypal.manifest
 playback_recordings = paypal.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
--- a/testing/raptor/raptor/tests/raptor-tp6-2.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-2.ini
@@ -12,16 +12,17 @@ python3_win_manifest = python3{x64}.mani
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-2.manifest
 page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 page_timeout = 60000
 gecko_profile_interval = 1
 gecko_profile_entries = 14000000
+alert_on = fcp
 
 [raptor-tp6-docs-firefox]
 apps = firefox
 test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing
 playback_recordings = google-docs.mp
 measure = fnbpaint, fcp, hero, dcf
 hero = hero1
 
--- a/testing/raptor/raptor/tests/raptor-tp6-3.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-3.ini
@@ -14,28 +14,30 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
 gecko_profile_interval = 1
 gecko_profile_entries = 2000000
+alert_on = fcp, loadtime
 
 [raptor-tp6-imdb-firefox]
 apps = firefox
 test_url = https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2
 playback_recordings = imdb.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
 [raptor-tp6-imgur-firefox]
 apps = firefox
 test_url = https://imgur.com/gallery/m5tYJL6
 playback_recordings = imgur.mp
 measure = fnbpaint, fcp, dcf, ttfi
+alert_on = fcp
 
 [raptor-tp6-wikia-firefox]
 apps = firefox
 test_url = http://fandom.wikia.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase
 playback_recordings = wikia.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
 [raptor-tp6-imdb-chrome]
@@ -44,14 +46,15 @@ test_url = https://www.imdb.com/title/tt
 playback_recordings = imdb.mp
 measure = fcp, loadtime
 
 [raptor-tp6-imgur-chrome]
 apps = chrome
 test_url = https://imgur.com/gallery/m5tYJL6
 playback_recordings = imgur.mp
 measure = fcp
+alert_on = fcp
 
 [raptor-tp6-wikia-chrome]
 apps = chrome
 test_url = http://fandom.wikia.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase
 playback_recordings = wikia.mp
 measure = fcp, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6-4.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-4.ini
@@ -14,16 +14,17 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
 gecko_profile_interval = 1
 gecko_profile_entries = 2000000
+alert_on = fcp, loadtime
 
 [raptor-tp6-bing-firefox]
 apps = firefox
 test_url = https://www.bing.com/search?q=barack+obama
 playback_recordings = bing.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
 [raptor-tp6-yandex-firefox]
--- a/testing/raptor/raptor/tests/raptor-tp6-5.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-5.ini
@@ -14,33 +14,36 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
 gecko_profile_interval = 1
 gecko_profile_entries = 2000000
+alert_on = fcp, loadtime
 
 [raptor-tp6-apple-firefox]
 apps = firefox
 test_url = https://www.apple.com/macbook-pro/
 playback_recordings = apple.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
 [raptor-tp6-microsoft-firefox]
 apps = firefox
 test_url = https://www.microsoft.com/en-us/windows/get-windows-10
 page_timeout = 60000
 playback_recordings = microsoft.mp
 measure = fnbpaint, fcp, dcf, ttfi
+alert_on = fcp
 
 [raptor-tp6-apple-chrome]
 apps = chrome
 test_url = https://www.apple.com/macbook-pro/
 playback_recordings = apple.mp
 measure = fcp, loadtime
 
 [raptor-tp6-microsoft-chrome]
 apps = chrome
 test_url = https://www.microsoft.com/en-us/windows/get-windows-10
 playback_recordings = microsoft.mp
 measure = fcp
+alert_on = fcp
--- a/testing/raptor/raptor/tests/raptor-tp6-6.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-6.ini
@@ -14,16 +14,17 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
 gecko_profile_interval = 1
 gecko_profile_entries = 2000000
+alert_on = fcp, loadtime
 
 [raptor-tp6-reddit-firefox]
 apps = firefox
 test_url = https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/
 playback_recordings = reddit.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
 [raptor-tp6-yahoo-news-firefox]
--- a/testing/raptor/raptor/tests/raptor-tp6-7.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-7.ini
@@ -14,16 +14,17 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 60000
 gecko_profile_interval = 1
 gecko_profile_entries = 2000000
+alert_on = fcp, loadtime
 
 [raptor-tp6-instagram-firefox]
 apps = firefox
 test_url = https://www.instagram.com/
 playback_recordings = instagram.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
 [raptor-tp6-twitter-firefox]
--- a/testing/raptor/raptor/tests/raptor-tp6-8.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-8.ini
@@ -13,16 +13,17 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
 gecko_profile_interval = 1
 gecko_profile_entries = 2000000
+alert_on = fcp, loadtime
 
 [raptor-tp6-ebay-firefox]
 apps = firefox
 test_url = https://www.ebay.com/
 playback_pageset_manifest = mitmproxy-recordings-raptor-ebay.manifest
 playback_recordings = ebay.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
--- a/testing/raptor/raptor/tests/raptor-tp6-9.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-9.ini
@@ -13,16 +13,17 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 # TTI/TTFI can take a while on some pages, and requires at least 5 seconds
 # beyond typical pageload time
 page_timeout = 30000
 gecko_profile_interval = 1
 gecko_profile_entries = 2000000
+alert_on = fcp, loadtime
 
 [raptor-tp6-google-mail-firefox]
 apps = firefox
 test_url = https://mail.google.com/
 playback_pageset_manifest = mitmproxy-recordings-raptor-google-mail.manifest
 playback_recordings = google-mail.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
@@ -40,9 +41,9 @@ playback_pageset_manifest = mitmproxy-re
 playback_recordings = google-mail.mp
 measure = fcp, loadtime
 
 [raptor-tp6-pinterest-chrome]
 apps = chrome
 test_url = https://pinterest.com/
 playback_pageset_manifest = mitmproxy-recordings-raptor-pinterest.manifest
 playback_recordings = pinterest.mp
-measure = fcp, loadtime
\ No newline at end of file
+measure = fcp, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-1.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-1.ini
@@ -9,16 +9,17 @@ type =  pageload
 playback = mitmproxy-android
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
 python3_win_manifest = python3{x64}.manifest
 page_cycles = 15
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 page_timeout = 60000
+alert_on = fcp, loadtime
 
 [raptor-tp6m-amazon-geckoview]
 apps = geckoview
 test_url = https://www.amazon.com
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon.manifest
 playback_recordings = android-amazon.mp
 measure = fnbpaint, fcp, dcf, ttfi, loadtime
 
--- a/testing/raptor/test/test_manifest.py
+++ b/testing/raptor/test/test_manifest.py
@@ -22,30 +22,32 @@ sys.path.insert(0, raptor_dir)
 from manifest import get_browser_test_list, validate_test_ini, get_raptor_test_list
 
 
 # some test details (test INIs)
 VALID_MANIFESTS = [{'apps': 'firefox',
                     'type': 'pageload',
                     'page_cycles': 25,
                     'test_url': 'http://www.test-url/goes/here',
-                    'measure': 'fnbpaint, dcf',
+                    'measure': 'fnbpaint, fcp',
+                    'alert_on': 'fcp',
                     'unit': 'ms',
                     'lower_is_better': True,
                     'alert_threshold': 2.0,
                     'playback': 'mitmproxy',
                     'playback_binary_manifest': 'binary.manifest',
                     'playback_pageset_manifest': 'pageset.manifest',
                     'playback_recordings': 'recorded_site.mp',
                     'python3_win_manifest': 'py3.manifest',
                     'manifest': 'valid_details_1'},
                    {'apps': 'chrome',
                     'type': 'benchmark',
                     'page_cycles': 5,
                     'test_url': 'http://www.test-url/goes/here',
+                    'measure': 'fcp',
                     'unit': 'score',
                     'lower_is_better': False,
                     'alert_threshold': 2.0,
                     'manifest': 'valid_details_2'}]
 
 INVALID_MANIFESTS = [{'apps': 'firefox',
                       'type': 'pageload',
                       'page_cycles': 25,
@@ -58,22 +60,37 @@ INVALID_MANIFESTS = [{'apps': 'firefox',
                       'playback_pageset_manifest': 'pageset.manifest',
                       'playback_recordings': 'recorded_site.mp',
                       'python3_win_manifest': 'py3.manifest',
                       'manifest': 'invalid_details_1'},
                      {'apps': 'chrome',
                       'type': 'pageload',
                       'page_cycles': 25,
                       'test_url': 'http://www.test-url/goes/here',
-                      'measure': 'fnbpaint, dcf',
+                      'measure': 'fnbpaint, fcp',
                       'unit': 'ms',
                       'lower_is_better': True,
                       'alert_threshold': 2.0,
                       'playback': 'mitmproxy',
-                      'manifest': 'invalid_details_2'}]
+                      'manifest': 'invalid_details_2'},
+                     {'apps': 'firefox',
+                      'type': 'pageload',
+                      'page_cycles': 25,
+                      'test_url': 'http://www.test-url/goes/here',
+                      'measure': 'fnbpaint, fcp',
+                      'alert_on': 'nope',
+                      'unit': 'ms',
+                      'lower_is_better': True,
+                      'alert_threshold': 2.0,
+                      'playback': 'mitmproxy',
+                      'playback_binary_manifest': 'binary.manifest',
+                      'playback_pageset_manifest': 'pageset.manifest',
+                      'playback_recordings': 'recorded_site.mp',
+                      'python3_win_manifest': 'py3.manifest',
+                      'manifest': 'invalid_details_3'}]
 
 
 @pytest.mark.parametrize('app', ['firefox', 'chrome', 'geckoview'])
 def test_get_browser_test_list(app):
     test_list = get_browser_test_list(app)
     assert len(test_list) > 0