Bug 1537944. Add conditioned-profile support for desktop Firefox in Raptor. r=perftest-reviewers,tarek,sparky, ?tarek
authorStephen Donner <sdonner@mozilla.com>
Fri, 15 Nov 2019 14:59:21 +0000
changeset 502179 6d7fa567184ed7fe82d21e0f629c09d7f285f717
parent 502178 b80efaa3e184d98e9ca691a4a35c0effc20d4573
child 502180 ce76d83dde22c957c75a8f5ab8161170fa762923
push id114172
push userdluca@mozilla.com
push dateTue, 19 Nov 2019 11:31:10 +0000
treeherdermozilla-inbound@b5c5ba07d3db [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, tarek, sparky
bugs1537944
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1537944. Add conditioned-profile support for desktop Firefox in Raptor. r=perftest-reviewers,tarek,sparky, ?tarek Differential Revision: https://phabricator.services.mozilla.com/D49970
taskcluster/ci/test/raptor.yml
taskcluster/taskgraph/transforms/raptor.py
testing/condprofile/condprof/util.py
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/raptor.py
testing/raptor/raptor/results.py
testing/raptor/requirements.txt
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -42,16 +42,17 @@ job-defaults:
                 default:
                     - raptor/linux_config.py
 
 raptor-tp6-1-firefox:
     description: "Raptor tp6-1 on Firefox"
     variants: ["fission"]
     try-name: raptor-tp6-1-firefox
     treeherder-symbol: Rap(tp6-1)
+    condprof: True
     mozharness:
         extra-options:
             - --test=raptor-tp6-1
 
 raptor-tp6-1-firefox-profiling:
     description: "Raptor tp6-1 on Firefox with Gecko Profiling"
     try-name: raptor-tp6-1-firefox-profiling
     treeherder-symbol: Rap-Prof(tp6-1)
@@ -683,16 +684,17 @@ raptor-youtube-playback-h264-power-firef
             - --test=raptor-youtube-playback-h264
             - --power-test
 
 raptor-tp6-1-firefox-cold:
     description: "Raptor tp6-1 cold page-load on Firefox"
     try-name: raptor-tp6-1-firefox-cold
     treeherder-symbol: Rap(tp6-c-1)
     tier: 2
+    condprof: True
     mozharness:
         extra-options:
             - --test=raptor-tp6-1-cold
 
 raptor-tp6-2-firefox-cold:
     description: "Raptor tp6-2 cold page-load on Firefox"
     try-name: raptor-tp6-2-firefox-cold
     treeherder-symbol: Rap(tp6-c-2)
--- a/taskcluster/taskgraph/transforms/raptor.py
+++ b/taskcluster/taskgraph/transforms/raptor.py
@@ -35,16 +35,20 @@ raptor_description_schema = Schema({
     Optional('binary-path'): optionally_keyed_by(
         'app',
         basestring
     ),
     Optional('pageload'): optionally_keyed_by(
         'test-platform', 'app',
         Any('cold', 'warm', 'both'),
     ),
+    Optional('condprof'): optionally_keyed_by(
+        'app',
+        bool,
+    ),
     # Configs defined in the 'test_description_schema'.
     Optional('max-run-time'): optionally_keyed_by(
         'app',
         test_description_schema['max-run-time']
     ),
     Optional('run-on-projects'): optionally_keyed_by(
         'app',
         test_description_schema['run-on-projects']
@@ -116,16 +120,17 @@ def split_apps(config, tests):
                 atest['treeherder-symbol'] = join_symbol(group, symbol)
 
             yield atest
 
 
 @transforms.add
 def handle_keyed_by_app(config, tests):
     fields = [
+        'condprof',
         'variants',
         'limit-platforms',
         'activity',
         'binary-path',
         'pageload',
         'max-run-time',
         'run-on-projects',
         'target',
@@ -159,16 +164,41 @@ def split_pageload(config, tests):
 
         group, symbol = split_symbol(test['treeherder-symbol'])
         symbol += '-c'
         test['treeherder-symbol'] = join_symbol(group, symbol)
         yield test
 
 
 @transforms.add
+def build_condprof_tests(config, tests):
+    for test in tests:
+        if not test.pop('condprof', False):
+            yield test
+            continue
+
+        # Make condprof test
+        condprof_test = deepcopy(test)
+        yield test
+
+        extra_options = condprof_test.setdefault('mozharness', {}).setdefault('extra-options', [])
+        extra_options.append('--with-conditioned-profile')
+
+        group, symbol = split_symbol(condprof_test['treeherder-symbol'])
+        symbol += '-condprof'
+
+        condprof_test['description'] += " with condprof"
+        condprof_test['try-name'] += '-condprof'
+        condprof_test['test-name'] += '-condprof'
+        condprof_test['treeherder-symbol'] = join_symbol(group, symbol)
+
+        yield condprof_test
+
+
+@transforms.add
 def add_extra_options(config, tests):
     for test in tests:
         extra_options = test.setdefault('mozharness', {}).setdefault('extra-options', [])
 
         if test.pop('run-visual-metrics', False):
             extra_options.append('--browsertime-video')
             test['attributes']['run-visual-metrics'] = True
 
--- a/testing/condprofile/condprof/util.py
+++ b/testing/condprofile/condprof/util.py
@@ -70,21 +70,23 @@ class NullLogger:
 
 def get_logger():
     global _LOGGER
     if _LOGGER is not None:
         return _LOGGER
 
     if sys.version_info.major == 3:
         # plugging the logger into arsenic
-        from arsenic import connection
-        from structlog import wrap_logger
-
-        logger = wrap_logger(NullLogger(), processors=[])
-        connection.log = logger
+        try:
+            from arsenic import connection
+            from structlog import wrap_logger
+            logger = wrap_logger(NullLogger(), processors=[])
+            connection.log = logger
+        except ImportError:
+            logger = NullLogger()
     else:
         # on python 2, just using the plain logger
         logger = NullLogger()
 
     _LOGGER = logger
     return _LOGGER
 
 
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -128,16 +128,22 @@ class Raptor(TestingMixin, MercurialScri
           "help": "Extra options to Raptor."
           }],
         [["--enable-webrender"], {
             "action": "store_true",
             "dest": "enable_webrender",
             "default": False,
             "help": "Enable the WebRender compositor in Gecko.",
         }],
+        [["--with-conditioned-profile"], {
+            "action": "store_true",
+            "dest": "with_conditioned_profile",
+            "default": False,
+            "help": "Run using the conditioned profile.",
+        }],
         [["--geckoProfile"], {
             "dest": "gecko_profile",
             "action": "store_true",
             "default": False,
             "help": argparse.SUPPRESS
         }],
         [["--geckoProfileInterval"], {
             "dest": "gecko_profile_interval",
@@ -485,16 +491,18 @@ class Raptor(TestingMixin, MercurialScri
         if self.config.get('power_test', False):
             options.extend(['--power-test'])
         if self.config.get('memory_test', False):
             options.extend(['--memory-test'])
         if self.config.get('cpu_test', False):
             options.extend(['--cpu-test'])
         if self.config.get('enable_webrender', False):
             options.extend(['--enable-webrender'])
+        if self.config.get('with_conditioned_profile', False):
+            options.extend(['--with-conditioned-profile'])
 
         for (arg,), details in Raptor.browsertime_options:
             # Allow overriding defaults on the `./mach raptor-test ...` command-line
             value = self.config.get(details['dest'])
             if value and arg not in self.config.get("raptor_cmd_line_args", []):
                 if isinstance(value, basestring):
                     options.extend([arg, os.path.expandvars(value)])
                 else:
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -135,16 +135,19 @@ def create_parser(mach_interface=False):
     add_arg('--print-tests', action=_PrintTests,
             help="Print all available Raptor tests")
     add_arg('--debug-mode', dest="debug_mode", action="store_true",
             help="Run Raptor in debug mode (open browser console, limited page-cycles, etc.)")
     add_arg('--disable-e10s', dest="e10s", action="store_false", default=True,
             help="Run without multiple processes (e10s).")
     add_arg('--enable-webrender', dest="enable_webrender", action="store_true", default=False,
             help="Enable the WebRender compositor in Gecko.")
+    add_arg('--with-conditioned-profile', dest="with_conditioned_profile", action="store_true",
+            default=False,
+            help="Run Raptor tests with a conditioned profile.")
     if not mach_interface:
         add_arg('--run-local', dest="run_local", default=False, action="store_true",
                 help="Flag which indicates if Raptor is running locally or in production")
         add_arg('--obj-path', dest="obj_path", default=None,
                 help="Browser-build obj_path (received when running in production)")
     add_arg('--noinstall', dest="noinstall", default=False, action="store_true",
             help="Flag which indicates if Raptor should not offer to install Android APK.")
     add_arg('--installerpath', dest="installerpath", default=None, type=str,
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -21,16 +21,18 @@ import time
 
 import requests
 
 import mozcrash
 import mozinfo
 import mozprocess
 import mozproxy.utils as mpu
 import mozversion
+from condprof.client import get_profile
+from condprof.util import get_current_platform
 from logger.logger import RaptorLogger
 from mozdevice import ADBDevice
 from mozlog import commandline
 from mozpower import MozPower
 from mozprofile import create_profile
 from mozproxy import get_playback
 from mozrunner import runners
 
@@ -98,17 +100,17 @@ either Raptor or browsertime."""
     __metaclass__ = ABCMeta
 
     def __init__(self, app, binary, run_local=False, noinstall=False,
                  obj_path=None, profile_class=None, installerpath=None,
                  gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
                  symbols_path=None, host=None, power_test=False, cpu_test=False, memory_test=False,
                  is_release_build=False, debug_mode=False, post_startup_delay=None,
                  interrupt_handler=None, e10s=True, enable_webrender=False,
-                 results_handler_class=RaptorResultsHandler,
+                 results_handler_class=RaptorResultsHandler, with_conditioned_profile=False,
                  **kwargs):
 
         # Override the magic --host HOST_IP with the value of the environment variable.
         if host == 'HOST_IP':
             host = os.environ['HOST_IP']
 
         self.config = {
             'app': app,
@@ -124,16 +126,17 @@ either Raptor or browsertime."""
             'host': host,
             'power_test': power_test,
             'memory_test': memory_test,
             'cpu_test': cpu_test,
             'is_release_build': is_release_build,
             'enable_control_server_wait': memory_test or cpu_test,
             'e10s': e10s,
             'enable_webrender': enable_webrender,
+            'with_conditioned_profile': with_conditioned_profile,
         }
         # We can never use e10s on fennec
         if self.config['app'] == 'fennec':
             self.config['e10s'] = False
 
         self.browser_name = None
         self.browser_version = None
 
@@ -141,16 +144,17 @@ either Raptor or browsertime."""
         self.installerpath = installerpath
         self.playback = None
         self.benchmark = None
         self.benchmark_port = 0
         self.gecko_profiler = None
         self.post_startup_delay = post_startup_delay
         self.device = None
         self.profile_class = profile_class or app
+        self.conditioned_profile_dir = None
         self.firefox_android_apps = FIREFOX_ANDROID_APPS
         self.interrupt_handler = interrupt_handler
         self.results_handler = results_handler_class(**self.config)
 
         self.browser_name, self.browser_version = self.get_browser_meta()
 
         browser_name, browser_version = self.get_browser_meta()
         self.results_handler.add_browser_meta(self.config['app'], browser_version)
@@ -163,18 +167,51 @@ either Raptor or browsertime."""
             self.post_startup_delay = min(self.post_startup_delay, 3000)
             LOG.info("debug-mode enabled, reducing post-browser startup pause to %d ms"
                      % self.post_startup_delay)
 
         LOG.info("main raptor init, config is: %s" % str(self.config))
 
         self.build_browser_profile()
 
+    def get_conditioned_profile(self):
+        """Downloads a platform-specific conditioned profile, using the
+        condprofile client API; returns a self.conditioned_profile_dir"""
+
+        # create a temp file to help ensure uniqueness
+        temp_download_dir = tempfile.mkdtemp()
+        LOG.info("Making temp_download_dir from inside get_conditioned_profile {}"
+                 .format(temp_download_dir))
+        platform = get_current_platform()
+        cond_prof_target_dir = get_profile(temp_download_dir, platform, "cold")
+        LOG.info("temp_download_dir is: {}".format(temp_download_dir))
+        LOG.info("cond_prof_target_dir is: {}".format(cond_prof_target_dir))
+
+        self.conditioned_profile_dir = os.path.join(temp_download_dir, cond_prof_target_dir)
+        if not os.path.exists(cond_prof_target_dir):
+            LOG.critical("Can't find target_dir {}, from get_profile()"
+                         "temp_download_dir {}, platform {}, cold"
+                         .format(cond_prof_target_dir, temp_download_dir, platform))
+            raise OSError
+
+        LOG.info("self.conditioned_profile_dir is now set: {}"
+                 .format(self.conditioned_profile_dir))
+        shutil.rmtree(temp_download_dir)
+
+        return self.conditioned_profile_dir
+
     def build_browser_profile(self):
-        self.profile = create_profile(self.profile_class)
+        # if --with-conditioned-profile was passed in via the commandline,
+        # we need to fetch and use a conditioned profile
+        if self.config['with_conditioned_profile']:
+            self.get_conditioned_profile()
+            self.profile = create_profile(self.profile_class, profile=self.conditioned_profile_dir)
+        else:
+            # have mozprofile create a new profile for us
+            self.profile = create_profile(self.profile_class)
 
         # Merge extra profile data from testing/profiles
         with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
             base_profiles = json.load(fh)['raptor']
 
         for profile in base_profiles:
             path = os.path.join(self.profile_data_dir, profile)
             LOG.info("Merging profile: {}".format(path))
@@ -806,16 +843,17 @@ class Raptor(Perftest):
         super(Raptor, self).__init__(*args, **kwargs)
 
         # set up the results handler
         self.results_handler = RaptorResultsHandler(
             gecko_profile=self.config.get('gecko_profile'),
             power_test=self.config.get('power_test'),
             cpu_test=self.config.get('cpu_test'),
             memory_test=self.config.get('memory_test'),
+            with_conditioned_profile=self.config['with_conditioned_profile']
         )
         browser_name, browser_version = self.get_browser_meta()
         self.results_handler.add_browser_meta(self.config['app'], browser_version)
 
         self.start_control_server()
 
     def run_test_setup(self, test):
         super(Raptor, self).run_test_setup(test)
@@ -1722,16 +1760,17 @@ def main(args=sys.argv[1:]):
                           memory_test=args.memory_test,
                           is_release_build=args.is_release_build,
                           debug_mode=args.debug_mode,
                           post_startup_delay=args.post_startup_delay,
                           activity=args.activity,
                           intent=args.intent,
                           interrupt_handler=SignalHandler(),
                           enable_webrender=args.enable_webrender,
+                          with_conditioned_profile=args.with_conditioned_profile,
                           )
 
     success = raptor.run_tests(raptor_test_list, raptor_test_names)
 
     if not success:
         # didn't get test results; test timed out or crashed, etc. we want job to fail
         LOG.critical("TEST-UNEXPECTED-FAIL: no raptor test results were found for %s" %
                      ', '.join(raptor_test_names))
--- a/testing/raptor/raptor/results.py
+++ b/testing/raptor/raptor/results.py
@@ -18,28 +18,30 @@ LOG = RaptorLogger(component='perftest-r
 
 
 class PerftestResultsHandler(object):
     """Abstract base class to handle perftest results"""
 
     __metaclass__ = ABCMeta
 
     def __init__(self, gecko_profile=False, power_test=False,
-                 cpu_test=False, memory_test=False, app=None, **kwargs):
+                 cpu_test=False, memory_test=False, app=None, with_conditioned_profile=False,
+                 **kwargs):
         self.gecko_profile = gecko_profile
         self.power_test = power_test
         self.cpu_test = cpu_test
         self.memory_test = memory_test
         self.app = app
         self.results = []
         self.page_timeout_list = []
         self.images = []
         self.supporting_data = None
         self.browser_version = None
         self.browser_name = None
+        self.with_conditioned_profile = with_conditioned_profile
 
     @abstractmethod
     def add(self, new_result_json):
         raise NotImplementedError()
 
     def add_browser_meta(self, browser_name, browser_version):
         # sets the browser metadata for the perfherder data
         self.browser_name = browser_name
@@ -168,16 +170,18 @@ class PerftestResultsHandler(object):
 
 class RaptorResultsHandler(PerftestResultsHandler):
     """Process Raptor results"""
 
     def add(self, new_result_json):
         # add to results
         LOG.info("received results in RaptorResultsHandler.add")
         new_result = RaptorTestResult(new_result_json)
+        if self.with_conditioned_profile:
+            new_result.extra_options.append('condprof')
         self.results.append(new_result)
 
     def summarize_and_output(self, test_config, tests, test_names):
         # summarize the result data, write to file and output PERFHERDER_DATA
         LOG.info("summarizing raptor test results")
         output = RaptorOutput(self.results, self.supporting_data, test_config['subtest_alert_on'])
         output.set_browser_meta(self.browser_name, self.browser_version)
         output.summarize(test_names)
--- a/testing/raptor/requirements.txt
+++ b/testing/raptor/requirements.txt
@@ -3,8 +3,9 @@ mozrunner ~= 7.0
 mozprofile ~= 2.1
 manifestparser >= 1.1
 wptserve ~= 1.4.0
 mozdevice >= 3.0.1
 mozproxy >= 1.0
 pyyaml ~= 3.1
 mozpower >= 1.0.0
 mozversion >= 1.0
+conditioned-profile >= 0.1