Bug 1531169 - Add the ability to run Raptor page-load tests against live sites; r=davehunt
authorRob Wood <rwood@mozilla.com>
Mon, 04 Mar 2019 20:40:51 +0000
changeset 462269 15d98a1290fa75aa119120ad3b591dff90956bc4
parent 462268 dd200b211b4adbbf149096dcf7341576384c5129
child 462270 adf5830d08b510ee37df21177ac71cd9c389660f
push id35646
push useraciure@mozilla.com
push dateTue, 05 Mar 2019 04:19:19 +0000
treeherdermozilla-central@7c5b913dc2e3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1531169
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1531169 - Add the ability to run Raptor page-load tests against live sites; r=davehunt Differential Revision: https://phabricator.services.mozilla.com/D21466
testing/raptor/raptor/manifest.py
testing/raptor/test/conftest.py
testing/raptor/test/test_manifest.py
testing/raptor/test/test_playback.py
--- a/testing/raptor/raptor/manifest.py
+++ b/testing/raptor/raptor/manifest.py
@@ -23,23 +23,39 @@ playback_settings = ['playback_binary_ma
 
 
 def filter_app(tests, values):
     for test in tests:
         if values["app"] in test['apps']:
             yield test
 
 
-def get_browser_test_list(browser_app):
+def filter_live_sites(tests, values):
+    # if a test uses live sites only allow it to run if running locally or on try
+    # this prevents inadvertently submitting live site data to perfherder
+    for test in tests:
+        if test.get("use_live_sites", "false") == "true":
+            # can run with live sites when running locally
+            if values["run_local"] is True:
+                yield test
+            # can run with live sites if running on try
+            if "hg.mozilla.org/try" in os.environ.get('GECKO_HEAD_REPOSITORY', 'n/a'):
+                yield test
+        else:
+            # not using live-sites so go ahead
+            yield test
+
+
+def get_browser_test_list(browser_app, run_local):
     LOG.info(raptor_ini)
     test_manifest = TestManifest([raptor_ini], strict=False)
-    info = {"app": browser_app}
+    info = {"app": browser_app, "run_local": run_local}
     return test_manifest.active_tests(exists=False,
                                       disabled=False,
-                                      filters=[filter_app],
+                                      filters=[filter_app, filter_live_sites],
                                       **info)
 
 
 def validate_test_ini(test_details):
     # validate all required test details were found in the test INI
     valid_settings = True
 
     for setting in required_settings:
@@ -91,17 +107,19 @@ def write_test_settings_json(args, test_
             "page_cycles": int(test_details['page_cycles']),
             "host": args.host,
         }
     }
 
     if test_details['type'] == "pageload":
         test_settings['raptor-options']['measure'] = {}
 
-        for m in [m.strip() for m in test_details['measure'].split(',')]:
+        # test_details['measure'] was already converted to a list in get_raptor_test_list below
+        # the 'hero=' line is still a raw string from the test INI
+        for m in test_details['measure']:
             test_settings['raptor-options']['measure'][m] = True
             if m == 'hero':
                 test_settings['raptor-options']['measure'][m] = [h.strip() for h in
                                                                  test_details['hero'].split(',')]
 
         if test_details.get("alert_on", None) is not None:
             # alert_on was already converted to list above
             test_settings['raptor-options']['alert_on'] = test_details['alert_on']
@@ -177,17 +195,17 @@ def get_raptor_test_list(args, oskey):
     Then look at the list of all available tests - each available test has a manifest
     name associated to it - and pull out all subtests whose manifest name matches
     the test name provided on the command line i.e. run all subtests in a specified ini.
 
     If no tests are found at all then the test name is invalid.
     '''
     tests_to_run = []
     # get list of all available tests for the browser we are testing against
-    available_tests = get_browser_test_list(args.app)
+    available_tests = get_browser_test_list(args.app, args.run_local)
 
     # look for single subtest that matches test name provided on cmd line
     for next_test in available_tests:
         if next_test['name'] == args.test:
             tests_to_run.append(next_test)
             break
 
     # no matches, so now look for all subtests that come from a test ini
@@ -221,16 +239,41 @@ def get_raptor_test_list(args, oskey):
             if int(next_test['page_cycles']) > max_page_cycles:
                 next_test['page_cycles'] = max_page_cycles
                 LOG.info("page-cycles set to %d" % next_test['page_cycles'])
         # if --page-timeout was provided on the command line, use that instead of INI
         if args.page_timeout is not None:
             LOG.info("setting page-timeout to %d as specified on cmd line" % args.page_timeout)
             next_test['page_timeout'] = args.page_timeout
 
+        if next_test.get('use_live_sites', "false") == "true":
+            # when using live sites we want to turn off playback
+            LOG.info("using live sites so turning playback off!")
+            next_test['playback'] = None
+            LOG.info("using live sites so appending '-live' to the test name")
+            next_test['name'] = next_test['name'] + "-live"
+            # we also want to increase the page timeout since may be longer live
+            next_test['page_timeout'] = 180000
+
+        # convert 'measure =' test INI line to list
+        if next_test.get('measure') is not None:
+            _measures = []
+            for m in [m.strip() for m in next_test['measure'].split(',')]:
+                # build the 'measures =' list
+                _measures.append(m)
+            next_test['measure'] = _measures
+
+            # if using live sites, don't measure hero element as it only exists in recordings
+            if 'hero' in next_test['measure'] and \
+               next_test.get('use_live_sites', "false") == "true":
+                # remove 'hero' from the 'measures =' list
+                next_test['measure'].remove('hero')
+                # remove the 'hero =' line since no longer measuring hero
+                del next_test['hero']
+
     # write out .json test setting files for the control server to read and send to web ext
     if len(tests_to_run) != 0:
         for test in tests_to_run:
             if validate_test_ini(test):
                 write_test_settings_json(args, test, oskey)
             else:
                 # test doesn't have valid settings, remove it from available list
                 LOG.info("test %s is not valid due to missing settings" % test['name'])
--- a/testing/raptor/test/conftest.py
+++ b/testing/raptor/test/conftest.py
@@ -72,17 +72,18 @@ def get_binary():
 def create_args():
     args = Namespace(app='firefox',
                      test='raptor-tp6-1',
                      binary='path/to/binary',
                      gecko_profile=False,
                      debug_mode=False,
                      page_cycles=None,
                      page_timeout=None,
-                     host=None)
+                     host=None,
+                     run_local=True)
 
     def inner(**kwargs):
         for next_arg in kwargs:
             print next_arg
             print kwargs[next_arg]
             setattr(args, next_arg, kwargs[next_arg])
         return args
 
--- a/testing/raptor/test/test_manifest.py
+++ b/testing/raptor/test/test_manifest.py
@@ -82,17 +82,17 @@ INVALID_MANIFESTS = [{'apps': 'firefox',
                       'playback_binary_manifest': 'binary.manifest',
                       'playback_pageset_manifest': 'pageset.manifest',
                       'playback_recordings': 'recorded_site.mp',
                       'manifest': 'invalid_details_3'}]
 
 
 @pytest.mark.parametrize('app', ['firefox', 'chrome', 'geckoview', 'refbrow', 'fenix'])
 def test_get_browser_test_list(app):
-    test_list = get_browser_test_list(app)
+    test_list = get_browser_test_list(app, run_local=True)
     if app != "fenix":
         assert len(test_list) > 0
     else:
         assert len(test_list) == 0
 
 
 @pytest.mark.parametrize('test_details', VALID_MANIFESTS)
 def test_validate_test_ini_valid(test_details):
@@ -122,23 +122,22 @@ def test_get_raptor_test_list_chrome(cre
                        test="raptor-speedometer")
 
     test_list = get_raptor_test_list(args, mozinfo.os)
     assert len(test_list) == 1
     assert test_list[0]['name'] == 'raptor-speedometer-chrome'
 
 
 def test_get_raptor_test_list_geckoview(create_args):
-    return
-#   args = create_args(app="geckoview",
-#                      test="raptor-unity-webgl")
-#
-#   test_list = get_raptor_test_list(args, mozinfo.os)
-#   assert len(test_list) == 1
-#   assert test_list[0]['name'] == 'raptor-unity-webgl-geckoview'
+    args = create_args(app="geckoview",
+                       test="raptor-unity-webgl")
+
+    test_list = get_raptor_test_list(args, mozinfo.os)
+    assert len(test_list) == 1
+    assert test_list[0]['name'] == 'raptor-unity-webgl-geckoview'
 
 
 def test_get_raptor_test_list_gecko_profiling(create_args):
     args = create_args(test="raptor-tp6-google-firefox",
                        gecko_profile=True)
 
     test_list = get_raptor_test_list(args, mozinfo.os)
     assert len(test_list) == 1
--- a/testing/raptor/test/test_playback.py
+++ b/testing/raptor/test/test_playback.py
@@ -22,18 +22,18 @@ def test_get_playback(get_binary):
     config['platform'] = mozinfo.os
     if 'win' in config['platform']:
         # this test is not yet supported on windows
         assert True
         return
     config['obj_path'] = os.path.dirname(get_binary('firefox'))
     config['playback_tool'] = 'mitmproxy'
     config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest'
-    config['playback_pageset_manifest'] = 'mitmproxy-playback-set.manifest'
-    config['playback_recordings'] = 'mitmproxy-recording-amazon.mp'
+    config['playback_pageset_manifest'] = 'mitmproxy-recordings-raptor-tp6-1.manifest'
+    config['playback_recordings'] = 'amazon.mp'
     config['binary'] = get_binary('firefox')
     config['run_local'] = run_local
 
     playback = get_playback(config)
     assert isinstance(playback, MitmproxyDesktop)
     playback.stop()