Bug 1628073 - Rewrite view_gecko_profile.py to open a URL; r=perftest-reviewers,sparky
authorGreg Tatum <gtatum@mozilla.com>
Fri, 22 May 2020 13:17:30 +0000
changeset 531611 448ea7e8873c622a1fa4f70f93503f4cedb84c39
parent 531610 d31e023899e51c9b09a5c9b382b29c15c12133ca
child 531612 ed24048652900853d41d71698a4f92f45cc240f3
push id37441
push userapavel@mozilla.com
push dateFri, 22 May 2020 21:38:53 +0000
treeherdermozilla-central@d6abd35b54ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, sparky
bugs1628073
milestone78.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1628073 - Rewrite view_gecko_profile.py to open a URL; r=perftest-reviewers,sparky The current implementation opens the built Firefox in the objdir. This is not optimal as the built Firefox is not really great for viewing files in. The build could be broken. With this patch, the profiles will instead be opened in the users default browser. Differential Revision: https://phabricator.services.mozilla.com/D70089
testing/raptor/raptor/raptor.py
testing/raptor/raptor/utils.py
testing/talos/talos/run_tests.py
testing/tools/view_gecko_profile/README.txt
testing/tools/view_gecko_profile/requirements.txt
testing/tools/view_gecko_profile/view_gecko_profile.py
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -27,17 +27,17 @@ except ImportError:
 from mozlog import commandline
 from mozprofile.cli import parse_preferences
 
 from browsertime import BrowsertimeDesktop, BrowsertimeAndroid
 from cmdline import parse_args, CHROMIUM_DISTROS
 from logger.logger import RaptorLogger
 from manifest import get_raptor_test_list
 from signal_handler import SignalHandler
-from utils import view_gecko_profile
+from utils import view_gecko_profile_from_raptor
 from webextension import (
     WebExtensionFirefox,
     WebExtensionDesktopChrome,
     WebExtensionAndroid,
 )
 
 LOG = RaptorLogger(component="raptor-main")
 
@@ -189,13 +189,13 @@ def main(args=sys.argv[1:]):
     # when running raptor locally with gecko profiling on, use the view-gecko-profile
     # tool to automatically load the latest gecko profile in profiler.firefox.com
     if args.gecko_profile and args.run_local:
         if os.environ.get("DISABLE_PROFILE_LAUNCH", "0") == "1":
             LOG.info(
                 "Not launching profiler.firefox.com because DISABLE_PROFILE_LAUNCH=1"
             )
         else:
-            view_gecko_profile(args.binary)
+            view_gecko_profile_from_raptor()
 
 
 if __name__ == "__main__":
     main()
--- a/testing/raptor/raptor/utils.py
+++ b/testing/raptor/raptor/utils.py
@@ -55,30 +55,23 @@ def transform_subtest(str_to_transform, 
     """Transform subtest name i.e. 'mitm4-linux-firefox-{subtest}.manifest'
     transforms to 'mitm4-linux-firefox-amazon.manifest'."""
     if '{subtest}' not in str_to_transform:
         return str_to_transform
 
     return str_to_transform.replace('{subtest}', subtest_name)
 
 
-def view_gecko_profile(ffox_bin):
-    # automatically load the latest talos gecko-profile archive in profiler.firefox.com
+def view_gecko_profile_from_raptor():
+    # automatically load the latest raptor gecko-profile archive in profiler.firefox.com
     LOG_GECKO = RaptorLogger(component='raptor-view-gecko-profile')
 
-    if sys.platform.startswith('win') and not ffox_bin.endswith(".exe"):
-        ffox_bin = ffox_bin + ".exe"
-
-    if not os.path.exists(ffox_bin):
-        LOG_GECKO.info("unable to find Firefox bin, cannot launch view-gecko-profile")
-        return
-
     profile_zip = os.environ.get('RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE', None)
     if profile_zip is None or not os.path.exists(profile_zip):
-        LOG_GECKO.info("No local talos gecko profiles were found so not "
+        LOG_GECKO.info("No local raptor gecko profiles were found so not "
                        "launching profiler.firefox.com")
         return
 
     # need the view-gecko-profile tool, it's in repo/testing/tools
     repo_dir = os.environ.get('MOZ_DEVELOPER_REPO_DIR', None)
     if repo_dir is None:
         LOG_GECKO.info("unable to find MOZ_DEVELOPER_REPO_DIR, can't launch view-gecko-profile")
         return
@@ -86,28 +79,25 @@ def view_gecko_profile(ffox_bin):
     view_gp = os.path.join(repo_dir, 'testing', 'tools',
                            'view_gecko_profile', 'view_gecko_profile.py')
     if not os.path.exists(view_gp):
         LOG_GECKO.info("unable to find the view-gecko-profile tool, cannot launch it")
         return
 
     command = [sys.executable,
                view_gp,
-               '-b', ffox_bin,
                '-p', profile_zip]
 
     LOG_GECKO.info('Auto-loading this profile in perfhtml.io: %s' % profile_zip)
     LOG_GECKO.info(command)
 
     # if the view-gecko-profile tool fails to launch for some reason, we don't
-    # want to crash talos! just dump error and finsh up talos as usual
+    # want to crash talos! just dump error and finish up talos as usual
     try:
-        view_profile = subprocess.Popen(command,
-                                        stdout=subprocess.PIPE,
-                                        stderr=subprocess.PIPE)
+        view_profile = subprocess.Popen(command)
         # that will leave it running in own instance and let talos finish up
     except Exception as e:
         LOG_GECKO.info("failed to launch view-gecko-profile tool, exeption: %s" % e)
         return
 
     time.sleep(5)
     ret = view_profile.poll()
     if ret is None:
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -335,32 +335,24 @@ function FindProxyForURL(url, host) {
                   % (results_urls['output_urls']))
 
     # when running talos locally with gecko profiling on, use the view-gecko-profile
     # tool to automatically load the latest gecko profile in profiler.firefox.com
     if config['gecko_profile'] and browser_config['develop']:
         if os.environ.get('DISABLE_PROFILE_LAUNCH', '0') == '1':
             LOG.info("Not launching profiler.firefox.com because DISABLE_PROFILE_LAUNCH=1")
         else:
-            view_gecko_profile(config['browser_path'])
+            view_gecko_profile_from_talos()
 
     # we will stop running tests on a failed test, or we will return 0 for
     # green
     return 0
 
 
-def view_gecko_profile(ffox_bin):
-    # automatically load the latest talos gecko-profile archive in profiler.firefox.com
-    if sys.platform.startswith('win') and not ffox_bin.endswith(".exe"):
-        ffox_bin = ffox_bin + ".exe"
-
-    if not os.path.exists(ffox_bin):
-        LOG.info("unable to find Firefox bin, cannot launch view-gecko-profile")
-        return
-
+def view_gecko_profile_from_talos():
     profile_zip = os.environ.get('TALOS_LATEST_GECKO_PROFILE_ARCHIVE', None)
     if profile_zip is None or not os.path.exists(profile_zip):
         LOG.info("No local talos gecko profiles were found so not launching profiler.firefox.com")
         return
 
     # need the view-gecko-profile tool, it's in repo/testing/tools
     repo_dir = os.environ.get('MOZ_DEVELOPER_REPO_DIR', None)
     if repo_dir is None:
@@ -370,28 +362,25 @@ def view_gecko_profile(ffox_bin):
     view_gp = os.path.join(repo_dir, 'testing', 'tools',
                            'view_gecko_profile', 'view_gecko_profile.py')
     if not os.path.exists(view_gp):
         LOG.info("unable to find the view-gecko-profile tool, cannot launch it")
         return
 
     command = ['python',
                view_gp,
-               '-b', ffox_bin,
                '-p', profile_zip]
 
-    LOG.info('Auto-loading this profile in perfhtml.io: %s' % profile_zip)
-    LOG.info(command)
+    LOG.info('Auto-loading this profile in profiler.firefox.com: %s' % profile_zip)
+    LOG.info(' '.join(command))
 
     # if the view-gecko-profile tool fails to launch for some reason, we don't
     # want to crash talos! just dump error and finsh up talos as usual
     try:
-        view_profile = subprocess.Popen(command,
-                                        stdout=subprocess.PIPE,
-                                        stderr=subprocess.PIPE)
+        view_profile = subprocess.Popen(command)
         # that will leave it running in own instance and let talos finish up
     except Exception as e:
         LOG.info("failed to launch view-gecko-profile tool, exeption: %s" % e)
         return
 
     time.sleep(5)
     ret = view_profile.poll()
     if ret is None:
--- a/testing/tools/view_gecko_profile/README.txt
+++ b/testing/tools/view_gecko_profile/README.txt
@@ -5,17 +5,13 @@ cd testing/tools/view_gecko_profile
 virtualenv venv
 
 source venv/bin/activate
 
 pip install -r requirements.txt
 
 Then the command line:
 
-python view_gecko_profile.py -b <path to browser binary> -p <path to gecko_profile.zip>
-
-i.e. For Firefox:
+python view_gecko_profile.py -p <path to gecko_profile.zip>
 
-python view_gecko_profile.py -b "/Users/rwood/mozilla-unified/obj-x86_64-apple-darwin17.4.0/dist/Nightly.app/Contents/MacOS/firefox" -p /Users/rwood/mozilla-unified/testing/mozharness/build/blobber_upload_dir/profile_damp.zip
+i.e.:
 
-i.e. For Chrome:
-
-python view_gecko_profile.py -b "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" -p /Users/rwood/mozilla-unified/testing/mozharness/build/blobber_upload_dir/profile_damp.zip
+python view_gecko_profile.py -p /Users/rwood/mozilla-unified/testing/mozharness/build/blobber_upload_dir/profile_damp.zip
--- a/testing/tools/view_gecko_profile/requirements.txt
+++ b/testing/tools/view_gecko_profile/requirements.txt
@@ -1,2 +1,1 @@
-mozrunner ~= 7.0
-wptserve ~= 1.4.0
+mozlog==6.0
--- a/testing/tools/view_gecko_profile/view_gecko_profile.py
+++ b/testing/tools/view_gecko_profile/view_gecko_profile.py
@@ -1,185 +1,120 @@
 #!/usr/bin/env python
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
+import sys
+
 
 import argparse
 import json
 import os
 import socket
 import sys
 import time
 import urllib
 from threading import Thread
 
 from mozlog import commandline, get_default_logger
 from mozlog.commandline import add_logging_group
-from mozrunner import runners
 
-from wptserve import server, handlers
+import SocketServer
+import SimpleHTTPServer
 
 here = os.path.abspath(os.path.dirname(__file__))
 
+class ProfileServingHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    """Extends the basic SimpleHTTPRequestHandler (which serves a directory
+    of files) to include request headers required by profiler.firefox.com"""
+    def end_headers(self):
+        self.send_header("Access-Control-Allow-Origin", "https://profiler.firefox.com")
+        SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self)
+
 
 class ViewGeckoProfile(object):
     """Container class for ViewGeckoProfile"""
 
-    def __init__(self, browser_binary, gecko_profile_zip):
+    def __init__(self, gecko_profile_data_path):
         self.log = get_default_logger(component='view-gecko-profile')
-        self.browser_bin = browser_binary
-        self.gecko_profile_zip = gecko_profile_zip
-        self.gecko_profile_dir = os.path.dirname(gecko_profile_zip)
-        self.perfhtmlio_url = "https://profiler.firefox.com/from-url/"
+        self.gecko_profile_data_path = gecko_profile_data_path
+        self.gecko_profile_dir = os.path.dirname(gecko_profile_data_path)
+        self.profiler_url = "https://profiler.firefox.com/from-url/"
         self.httpd = None
-        self.host = None
+        self.host = '127.0.0.1'
         self.port = None
 
-        # Create the runner
-        self.output_handler = OutputHandler()
-        process_args = {
-            'processOutputLine': [self.output_handler],
-        }
-        runner_cls = runners['firefox']
-        self.runner = runner_cls(
-            browser_binary, profile=None, process_args=process_args)
-
-    def start_http_server(self):
-        self.write_server_headers()
-
+    def setup_http_server(self):
         # pick a free port
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.bind(('', 0))
-        port = sock.getsockname()[1]
+        self.port = sock.getsockname()[1]
         sock.close()
-        _webserver = '127.0.0.1:%d' % port
-
-        self.httpd = self.setup_webserver(_webserver)
-        self.httpd.start()
 
-    def write_server_headers(self):
-        # to add specific headers for serving files via wptserve, write out a headers dir file
-        # see http://wptserve.readthedocs.io/en/latest/handlers.html#file-handlers
-        self.log.info("writing wptserve headers file")
-        headers_file = os.path.join(self.gecko_profile_dir, '__dir__.headers')
-        file = open(headers_file, 'w')
-        file.write("Access-Control-Allow-Origin: *")
-        file.close()
-        self.log.info("wrote wpt headers file: %s" % headers_file)
+        os.chdir(self.gecko_profile_dir)
+        self.httpd = SocketServer.TCPServer((self.host, self.port), ProfileServingHTTPRequestHandler)
+        self.log.info("File server started at: %s:%s" % (self.host, self.port))
 
-    def setup_webserver(self, webserver):
-        self.log.info("starting webserver on %r" % webserver)
-        self.host, self.port = webserver.split(':')
-
-        return server.WebTestHttpd(port=int(self.port), doc_root=self.gecko_profile_dir,
-                            routes=[("GET", "*", handlers.file_handler)])
+    def handle_single_request(self):
+        self.httpd.handle_request()
 
     def encode_url(self):
         # Encode url i.e.: https://profiler.firefox.com/from-url/http... (the profile_zip served locally)
-        url = 'http://' + self.host + ':' + self.port + '/' + os.path.basename(self.gecko_profile_zip)
+        url = "http://{}:{}/{}".format(self.host, self.port,
+                                       os.path.basename(self.path_to_gecko_profile_data))
         self.log.info("raw url is:")
         self.log.info(url)
         encoded_url = urllib.quote(url, safe='')
         self.log.info('encoded url is:')
         self.log.info(encoded_url)
-        self.perfhtmlio_url = self.perfhtmlio_url + encoded_url
+        self.profiler_url = self.profiler_url + encoded_url
         self.log.info('full url is:')
-        self.log.info(self.perfhtmlio_url)
-
-    def start_browser_perfhtml(self):
-        self.log.info("Starting browser and opening the gecko profile zip in profiler.firefox.com...")
-
-        self.runner.cmdargs.append(self.perfhtmlio_url)
-        self.runner.start()
-
-        first_time = int(time.time()) * 1000
-        proc = self.runner.process_handler
-        self.output_handler.proc = proc
-
-        try:
-            self.runner.wait(timeout=None)
-        except Exception:
-            self.log.info("Failed to start browser")
-
-
-class OutputHandler(object):
-    def __init__(self):
-        self.proc = None
-        self.kill_thread = Thread(target=self.wait_for_quit)
-        self.kill_thread.daemon = True
-        self.log = get_default_logger(component='view_gecko_profile')
+        self.log.info(self.profiler_url)
 
-    def __call__(self, line):
-        if not line.strip():
-            return
-        line = line.decode('utf-8', errors='replace')
-
-        try:
-            data = json.loads(line)
-        except ValueError:
-            self.process_output(line)
-            return
-
-        if isinstance(data, dict) and 'action' in data:
-            self.log.log_raw(data)
-        else:
-            self.process_output(json.dumps(data))
-
-    def process_output(self, line):
-        self.log.process_output(self.proc.pid, line)
-
-    def wait_for_quit(self, timeout=5):
-        """Wait timeout seconds for the process to exit. If it hasn't
-        exited by then, kill it.
-        """
-        time.sleep(timeout)
-        if self.proc.poll() is None:
-            self.proc.kill()
+    def open_profile_in_browser(self):
+        # Open the file in the user's preferred browser.
+        self.log.info("Opening the profile data in profiler.firefox.com...")
+        import webbrowser
+        webbrowser.open_new_tab(self.profiler_url)
 
 
 def create_parser(mach_interface=False):
     parser = argparse.ArgumentParser()
     add_arg = parser.add_argument
 
-    add_arg('-b', '--binary', required=True, dest='binary',
-            help="path to browser executable")
     add_arg('-p', '--profile-zip', required=True, dest='profile_zip',
             help="path to the gecko profiles zip file to open in profiler.firefox.com")
 
     add_logging_group(parser)
     return parser
 
 
 def verify_options(parser, args):
     ctx = vars(args)
 
-    if not os.path.isfile(args.binary):
-        parser.error("{binary} does not exist!".format(**ctx))
-
     if not os.path.isfile(args.profile_zip):
         parser.error("{profile_zip} does not exist!".format(**ctx))
 
 
 def parse_args(argv=None):
     parser = create_parser()
     args = parser.parse_args(argv)
     verify_options(parser, args)
     return args
 
 
 def main(args=sys.argv[1:]):
     args = parse_args()
     commandline.setup_logging('view-gecko-profile', args, {'tbpl': sys.stdout})
     LOG = get_default_logger(component='view-gecko-profile')
 
-    view_gecko_profile = ViewGeckoProfile(args.binary, args.profile_zip)
+    view_gecko_profile = ViewGeckoProfile(args.profile_zip)
 
-    view_gecko_profile.start_http_server()
+    view_gecko_profile.setup_http_server()
     view_gecko_profile.encode_url()
-    view_gecko_profile.start_browser_perfhtml()
-
+    view_gecko_profile.open_profile_in_browser()
+    view_gecko_profile.handle_single_request()
 
 if __name__ == "__main__":
     main()