Bug 1573940 - Integrate mozpower into raptor desktop. r=perftest-reviewers,rwood
authorGregory Mierzwinski <gmierz2@outlook.com>
Fri, 23 Aug 2019 18:27:05 +0000
changeset 553390 1c8286c5178cea03fcc10712f002b79875bd14a0
parent 553389 06e0c353b3ef7b08c9bacc0f4c0eed30102de68c
child 553391 7ddf38b4b44c7d5fa7dd2ad925623953e94f69de
push id2165
push userffxbld-merge
push dateMon, 14 Oct 2019 16:30:58 +0000
treeherdermozilla-release@0eae18af659f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, rwood
bugs1573940
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1573940 - Integrate mozpower into raptor desktop. r=perftest-reviewers,rwood This patch integrates mozpower into raptor desktop testing. It can be used for MacOS power testing by supplying the `--power-test` command at the command line. Some changes to how the `--power-test` command is parsed are also made so that we don't check for the `--host` argument when it is supplied. Now, it is only checked when `--app` is an android browser, in the near future this `--host` argument will no longer be needed for power testing on android. Another change in this patch is the addition of the `self.artifact_dir` property which returns the current directory that should be used to output the artifacts. Power usage data is output into this artifact directory in a 'power-measurements' folder and this data is zipped when we are running in CI. Differential Revision: https://phabricator.services.mozilla.com/D42014
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/raptor.py
testing/raptor/requirements.txt
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 from __future__ import absolute_import, print_function
 
 import argparse
 import os
+import platform
 
 from mozlog.commandline import add_logging_group
 
 (FIREFOX,
  CHROME,
  CHROMIUM) = DESKTOP_APPS = ["firefox", "chrome", "chromium"]
 (FENNEC,
  GECKOVIEW,
@@ -79,19 +80,20 @@ def create_parser(mach_interface=False):
             help="Name of Android intent action used to launch the Android app."
             "i.e.: %s" % print_all_intents())
     add_arg('--host', dest='host',
             help="Hostname from which to serve URLs; defaults to 127.0.0.1. "
             "The value HOST_IP will cause the value of host to be "
             "loaded from the environment variable HOST_IP.",
             default='127.0.0.1')
     add_arg('--power-test', dest="power_test", action="store_true",
-            help="Use Raptor to measure power usage. Supported across GeckoView, "
-            "Fenix, Firefox (Fennec), and Reference Browsers."
-            "The host ip address must be specified via the --host command line argument.")
+            help="Use Raptor to measure power usage on Android browsers (Geckoview Example, "
+            "Fenix, Refbrow, and Fennec) as well as on Intel-based MacOS machines that have "
+            "Intel Power Gadget installed. The host ip address must be specified via the "
+            "--host command line argument if an android device/browser is being tested.")
     add_arg('--memory-test', dest="memory_test", action="store_true",
             help="Use Raptor to measure memory usage.")
     add_arg('--cpu-test', dest="cpu_test", action="store_true",
             help="Use Raptor to measure CPU usage. Currently supported for Android only.")
     add_arg('--is-release-build', dest="is_release_build", default=False,
             action='store_true',
             help="Whether the build is a release build which requires workarounds "
             "using MOZ_DISABLE_NONLOCAL_CONNECTIONS to support installing unsigned "
@@ -171,22 +173,25 @@ def verify_options(parser, args):
     if args.app in DESKTOP_APPS:
         if not os.path.isfile(args.binary):
             parser.error("{binary} does not exist!".format(**ctx))
 
     # if geckoProfile specified but not running on Firefox, not supported
     if args.gecko_profile is True and args.app != "firefox":
         parser.error("Gecko profiling is only supported when running Raptor on Firefox!")
 
-    # if --power-test specified, must be on geckoview/android with --host specified.
+    # if running power tests on geckoview/android, --host must be specified.
     if args.power_test:
-        if args.app not in ["fennec", "geckoview", "refbrow", "fenix"] \
-                or args.host in ('localhost', '127.0.0.1'):
-            parser.error("Power test is only supported when running Raptor on Firefox Android "
-                         "browsers when host is specified!")
+        if args.app in ["fennec", "geckoview", "refbrow", "fenix"]:
+            if args.host in ('localhost', '127.0.0.1'):
+                parser.error("When running power tests on Android browsers, the --host "
+                             "argument is required.")
+        elif platform.system().lower() not in ('darwin',):
+            parser.error("--power-test is only available on MacOS desktop machines, "
+                         "platform detected: %s." % platform.system().lower())
 
     if args.cpu_test:
         if args.app not in ["fennec", "geckoview", "refbrow", "fenix"]:
             parser.error("CPU test is only supported when running Raptor on Firefox Android "
                          "browsers!")
 
     if args.memory_test:
         if args.app not in ["fennec", "geckoview", "refbrow", "fenix"]:
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -18,16 +18,17 @@ import time
 
 import requests
 
 import mozcrash
 import mozinfo
 from logger.logger import RaptorLogger
 from mozdevice import ADBDevice
 from mozlog import commandline
+from mozpower import MozPower
 from mozprofile import create_profile
 from mozproxy import get_playback
 from mozrunner import runners
 
 # need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
 paths = [here]
 
@@ -165,16 +166,29 @@ either Raptor or browsertime."""
     @property
     def profile_data_dir(self):
         if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
             return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
         if build:
             return os.path.join(build.topsrcdir, 'testing', 'profiles')
         return os.path.join(here, 'profile_data')
 
+    @property
+    def artifact_dir(self):
+        artifact_dir = os.getcwd()
+        if self.config.get('run_local', False):
+            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
+                artifact_dir = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
+                                            'testing', 'mozharness', 'build')
+            else:
+                artifact_dir = here
+        elif os.getenv('MOZ_UPLOAD_DIR'):
+            artifact_dir = os.getenv('MOZ_UPLOAD_DIR')
+        return artifact_dir
+
     @abstractmethod
     def check_for_crashes(self):
         pass
 
     @abstractmethod
     def run_test_setup(self, test):
         LOG.info("starting test: %s" % test['name'])
 
@@ -205,23 +219,18 @@ either Raptor or browsertime."""
             self.gecko_profiler.symbolicate()
             # clean up the temp gecko profiling folders
             LOG.info("cleaning up after gecko profiling")
             self.gecko_profiler.clean()
 
     def process_results(self, test_names):
         # when running locally output results in build/raptor.json; when running
         # in production output to a local.json to be turned into tc job artifact
-        if self.config.get('run_local', False):
-            if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
-                raptor_json_path = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
-                                                'testing', 'mozharness', 'build', 'raptor.json')
-            else:
-                raptor_json_path = os.path.join(here, 'raptor.json')
-        else:
+        raptor_json_path = os.path.join(self.artifact_dir, 'raptor.json')
+        if not self.config.get('run_local', False):
             raptor_json_path = os.path.join(os.getcwd(), 'local.json')
 
         self.config['raptor_json_path'] = raptor_json_path
         return self.results_handler.summarize_and_output(self.config, test_names)
 
     @abstractmethod
     def clean_up(self):
         pass
@@ -532,21 +541,59 @@ class RaptorDesktop(Raptor):
         self.output_handler.proc = proc
 
         # give our control server the browser process so it can shut it down later
         self.control_server.browser_proc = proc
 
     def run_test(self, test, timeout):
         # tests will be run warm (i.e. NO browser restart between page-cycles)
         # unless otheriwse specified in the test INI by using 'cold = true'
+        mozpower_measurer = None
+        if self.config.get('power_test', False):
+            output_dir = os.path.join(self.artifact_dir, 'power-measurements')
+            test_dir = os.path.join(output_dir, test['name'].replace('/', '-').replace('\\', '-'))
+
+            try:
+                if not os.path.exists(output_dir):
+                    os.mkdir(output_dir)
+                if not os.path.exists(test_dir):
+                    os.mkdir(test_dir)
+            except Exception as e:
+                LOG.critical("Could not create directories to store power testing data.")
+                raise e
+
+            # Start power measurements with IPG creating a power usage log
+            # every 30 seconds with 1 data point per second (or a 1000 milli-
+            # second sampling rate).
+            mozpower_measurer = MozPower(
+                ipg_measure_duration=30,
+                sampling_rate=1000,
+                output_file_path=os.path.join(test_dir, 'power-usage')
+            )
+            mozpower_measurer.initialize_power_measurements()
+
         if test.get('cold', False) is True:
             self.__run_test_cold(test, timeout)
         else:
             self.__run_test_warm(test, timeout)
 
+        if mozpower_measurer:
+            mozpower_measurer.finalize_power_measurements(test_name=test['name'])
+            perfherder_data = mozpower_measurer.get_perfherder_data()
+
+            if not self.config.get('run_local', False):
+                # when not running locally, zip the data and delete the folder which
+                # was placed in the zip
+                power_data_path = os.path.join(self.artifact_dir, 'power-measurements')
+                shutil.make_archive(power_data_path + '.zip', 'zip', power_data_path)
+                shutil.rmtree(power_data_path)
+
+            self.control_server.submit_supporting_data(perfherder_data['utilization'])
+            self.control_server.submit_supporting_data(perfherder_data['power-usage'])
+
     def __run_test_cold(self, test, timeout):
         '''
         Run the Raptor test but restart the entire browser app between page-cycles.
 
         Note: For page-load tests, playback will only be started once - at the beginning of all
         browser cycles, and then stopped after all cycles are finished. That includes the import
         of the mozproxy ssl cert and turning on the browser proxy.
 
--- a/testing/raptor/requirements.txt
+++ b/testing/raptor/requirements.txt
@@ -1,8 +1,9 @@
 mozcrash ~= 1.0
 mozrunner ~= 7.0
 mozprofile ~= 2.1
 manifestparser >= 1.1
 wptserve ~= 1.4.0
 mozdevice >= 3.0.1
 mozproxy >= 1.0
 pyyaml ~= 3.1
+mozpower >= 1.0.0