Bug 1573940 - Integrate mozpower into raptor desktop. r=perftest-reviewers,rwood
authorGregory Mierzwinski <gmierz2@outlook.com>
Fri, 23 Aug 2019 18:27:05 +0000
changeset 553390 1c8286c5178cea03fcc10712f002b79875bd14a0
parent 553389 06e0c353b3ef7b08c9bacc0f4c0eed30102de68c
child 553391 7ddf38b4b44c7d5fa7dd2ad925623953e94f69de
push id2165
push userffxbld-merge
push dateMon, 14 Oct 2019 16:30:58 +0000
treeherdermozilla-release@0eae18af659f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, rwood
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1573940 - Integrate mozpower into raptor desktop. r=perftest-reviewers,rwood This patch integrates mozpower into raptor desktop testing. It can be used for MacOS power testing by supplying the `--power-test` command at the command line. Some changes to how the `--power-test` command is parsed are also made so that we don't check for the `--host` argument when it is supplied. Now, it is only checked when `--app` is an android browser, in the near future this `--host` argument will no longer be needed for power testing on android. Another change in this patch is the addition of the `self.artifact_dir` property which returns the current directory that should be used to output the artifacts. Power usage data is output into this artifact directory in a 'power-measurements' folder and this data is zipped when we are running in CI. Differential Revision: https://phabricator.services.mozilla.com/D42014
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 from __future__ import absolute_import, print_function
 import argparse
 import os
+import platform
 from mozlog.commandline import add_logging_group
  CHROMIUM) = DESKTOP_APPS = ["firefox", "chrome", "chromium"]
@@ -79,19 +80,20 @@ def create_parser(mach_interface=False):
             help="Name of Android intent action used to launch the Android app."
             "i.e.: %s" % print_all_intents())
     add_arg('--host', dest='host',
             help="Hostname from which to serve URLs; defaults to "
             "The value HOST_IP will cause the value of host to be "
             "loaded from the environment variable HOST_IP.",
     add_arg('--power-test', dest="power_test", action="store_true",
-            help="Use Raptor to measure power usage. Supported across GeckoView, "
-            "Fenix, Firefox (Fennec), and Reference Browsers."
-            "The host ip address must be specified via the --host command line argument.")
+            help="Use Raptor to measure power usage on Android browsers (Geckoview Example, "
+            "Fenix, Refbrow, and Fennec) as well as on Intel-based MacOS machines that have "
+            "Intel Power Gadget installed. The host ip address must be specified via the "
+            "--host command line argument if an android device/browser is being tested.")
     add_arg('--memory-test', dest="memory_test", action="store_true",
             help="Use Raptor to measure memory usage.")
     add_arg('--cpu-test', dest="cpu_test", action="store_true",
             help="Use Raptor to measure CPU usage. Currently supported for Android only.")
     add_arg('--is-release-build', dest="is_release_build", default=False,
             help="Whether the build is a release build which requires workarounds "
             "using MOZ_DISABLE_NONLOCAL_CONNECTIONS to support installing unsigned "
@@ -171,22 +173,25 @@ def verify_options(parser, args):
     if args.app in DESKTOP_APPS:
         if not os.path.isfile(args.binary):
             parser.error("{binary} does not exist!".format(**ctx))
     # if geckoProfile specified but not running on Firefox, not supported
     if args.gecko_profile is True and args.app != "firefox":
         parser.error("Gecko profiling is only supported when running Raptor on Firefox!")
-    # if --power-test specified, must be on geckoview/android with --host specified.
+    # if running power tests on geckoview/android, --host must be specified.
     if args.power_test:
-        if args.app not in ["fennec", "geckoview", "refbrow", "fenix"] \
-                or args.host in ('localhost', ''):
-            parser.error("Power test is only supported when running Raptor on Firefox Android "
-                         "browsers when host is specified!")
+        if args.app in ["fennec", "geckoview", "refbrow", "fenix"]:
+            if args.host in ('localhost', ''):
+                parser.error("When running power tests on Android browsers, the --host "
+                             "argument is required.")
+        elif platform.system().lower() not in ('darwin',):
+            parser.error("--power-test is only available on MacOS desktop machines, "
+                         "platform detected: %s." % platform.system().lower())
     if args.cpu_test:
         if args.app not in ["fennec", "geckoview", "refbrow", "fenix"]:
             parser.error("CPU test is only supported when running Raptor on Firefox Android "
     if args.memory_test:
         if args.app not in ["fennec", "geckoview", "refbrow", "fenix"]:
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -18,16 +18,17 @@ import time
 import requests
 import mozcrash
 import mozinfo
 from logger.logger import RaptorLogger
 from mozdevice import ADBDevice
 from mozlog import commandline
+from mozpower import MozPower
 from mozprofile import create_profile
 from mozproxy import get_playback
 from mozrunner import runners
 # need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
 paths = [here]
@@ -165,16 +166,29 @@ either Raptor or browsertime."""
     def profile_data_dir(self):
         if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
             return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
         if build:
             return os.path.join(build.topsrcdir, 'testing', 'profiles')
         return os.path.join(here, 'profile_data')
+    @property
+    def artifact_dir(self):
+        artifact_dir = os.getcwd()
+        if self.config.get('run_local', False):
+            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
+                artifact_dir = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
+                                            'testing', 'mozharness', 'build')
+            else:
+                artifact_dir = here
+        elif os.getenv('MOZ_UPLOAD_DIR'):
+            artifact_dir = os.getenv('MOZ_UPLOAD_DIR')
+        return artifact_dir
     def check_for_crashes(self):
     def run_test_setup(self, test):
         LOG.info("starting test: %s" % test['name'])
@@ -205,23 +219,18 @@ either Raptor or browsertime."""
             # clean up the temp gecko profiling folders
             LOG.info("cleaning up after gecko profiling")
     def process_results(self, test_names):
         # when running locally output results in build/raptor.json; when running
         # in production output to a local.json to be turned into tc job artifact
-        if self.config.get('run_local', False):
-            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
-                raptor_json_path = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
-                                                'testing', 'mozharness', 'build', 'raptor.json')
-            else:
-                raptor_json_path = os.path.join(here, 'raptor.json')
-        else:
+        raptor_json_path = os.path.join(self.artifact_dir, 'raptor.json')
+        if not self.config.get('run_local', False):
             raptor_json_path = os.path.join(os.getcwd(), 'local.json')
         self.config['raptor_json_path'] = raptor_json_path
         return self.results_handler.summarize_and_output(self.config, test_names)
     def clean_up(self):
@@ -532,21 +541,59 @@ class RaptorDesktop(Raptor):
         self.output_handler.proc = proc
         # give our control server the browser process so it can shut it down later
         self.control_server.browser_proc = proc
     def run_test(self, test, timeout):
         # tests will be run warm (i.e. NO browser restart between page-cycles)
         # unless otheriwse specified in the test INI by using 'cold = true'
+        mozpower_measurer = None
+        if self.config.get('power_test', False):
+            output_dir = os.path.join(self.artifact_dir, 'power-measurements')
+            test_dir = os.path.join(output_dir, test['name'].replace('/', '-').replace('\\', '-'))
+            try:
+                if not os.path.exists(output_dir):
+                    os.mkdir(output_dir)
+                if not os.path.exists(test_dir):
+                    os.mkdir(test_dir)
+            except Exception as e:
+                LOG.critical("Could not create directories to store power testing data.")
+                raise e
+            # Start power measurements with IPG creating a power usage log
+            # every 30 seconds with 1 data point per second (or a 1000 milli-
+            # second sampling rate).
+            mozpower_measurer = MozPower(
+                ipg_measure_duration=30,
+                sampling_rate=1000,
+                output_file_path=os.path.join(test_dir, 'power-usage')
+            )
+            mozpower_measurer.initialize_power_measurements()
         if test.get('cold', False) is True:
             self.__run_test_cold(test, timeout)
             self.__run_test_warm(test, timeout)
+        if mozpower_measurer:
+            mozpower_measurer.finalize_power_measurements(test_name=test['name'])
+            perfherder_data = mozpower_measurer.get_perfherder_data()
+            if not self.config.get('run_local', False):
+                # when not running locally, zip the data and delete the folder which
+                # was placed in the zip
+                power_data_path = os.path.join(self.artifact_dir, 'power-measurements')
+                shutil.make_archive(power_data_path + '.zip', 'zip', power_data_path)
+                shutil.rmtree(power_data_path)
+            self.control_server.submit_supporting_data(perfherder_data['utilization'])
+            self.control_server.submit_supporting_data(perfherder_data['power-usage'])
     def __run_test_cold(self, test, timeout):
         Run the Raptor test but restart the entire browser app between page-cycles.
         Note: For page-load tests, playback will only be started once - at the beginning of all
         browser cycles, and then stopped after all cycles are finished. That includes the import
         of the mozproxy ssl cert and turning on the browser proxy.
--- a/testing/raptor/requirements.txt
+++ b/testing/raptor/requirements.txt
@@ -1,8 +1,9 @@
 mozcrash ~= 1.0
 mozrunner ~= 7.0
 mozprofile ~= 2.1
 manifestparser >= 1.1
 wptserve ~= 1.4.0
 mozdevice >= 3.0.1
 mozproxy >= 1.0
 pyyaml ~= 3.1
+mozpower >= 1.0.0