Bug 1425308 - Save talos gecko profiles in a local 'latest-talos-profiles' folder draft
authorRob Wood <rwood@mozilla.com>
Wed, 10 Jan 2018 16:47:44 -0500
changeset 719290 c68aebb81fc5017ca5adfc32a078ab01c1f4e9de
parent 718696 68b7531613497d4278991e7e32fca8378f657534
child 745745 0fb6ca7119ec44e56c84bb55a9a588cbeb05fb91
push id95202
push userrwood@mozilla.com
push dateThu, 11 Jan 2018 20:03:15 +0000
bugs1425308
milestone59.0a1
Bug 1425308 - Save talos gecko profiles in a local 'latest-talos-profiles' folder Currently when running talos tests with gecko profiling on, the resulting profiles are zipped up (and uploaded as an artifact if running in production). This commit creates a 'gecko_profiles_out' folder when running locally, and stores the profiles there (as well as in the zip file). This will allow devs to easily view the gecko profiles from the latest talos test run, without having to find and unzip the profiles zip. MozReview-Commit-ID: CkDy3FZILvU
.hgignore
testing/talos/talos/config.py
testing/talos/talos/gecko_profile.py
testing/talos/talos/run_tests.py
--- a/.hgignore
+++ b/.hgignore
@@ -136,16 +136,17 @@ GPATH
 ^testing/talos/include/
 ^testing/talos/lib/
 ^testing/talos/talos/tests/tp5n.zip
 ^testing/talos/talos/tests/tp5n.tar.gz
 ^testing/talos/talos/tests/tp5n
 ^testing/talos/talos/tests/devtools/damp.manifest.develop
 ^talos-venv
 ^py3venv
+^testing/talos/talos/gecko_profiles_out/
 ^testing/talos/talos/mitmproxy/mitmdump
 ^testing/talos/talos/mitmproxy/mitmproxy
 ^testing/talos/talos/mitmproxy/mitmweb
 
 # Ignore talos speedometer files; source is copied from in-tree /third_party
 # into testing/talos/talos/tests/webkit/PerformanceTests/Speedometer when
 # talos speedometer test is run locally
 ^testing/talos/talos/tests/webkit/PerformanceTests/Speedometer
--- a/testing/talos/talos/config.py
+++ b/testing/talos/talos/config.py
@@ -426,16 +426,17 @@ def get_browser_config(config):
                 'test_timeout': 1200,
                 'xperf_path': None,
                 'error_filename': None,
                 'no_upload_results': False,
                 'enable_stylo': False,
                 'disable_stylo': False,
                 'stylothreads': 0,
                 'subtests': None,
+                'gecko_profiles_dir': '${talos}/gecko_profiles_out',
                 }
     browser_config = dict(title=config['title'])
     browser_config.update(dict([(i, config[i]) for i in required]))
     browser_config.update(dict([(i, config.get(i, j))
                           for i, j in optional.items()]))
     return browser_config
 
 
--- a/testing/talos/talos/gecko_profile.py
+++ b/testing/talos/talos/gecko_profile.py
@@ -61,16 +61,22 @@ class GeckoProfile(object):
 
         self.profiling_info = {
             "gecko_profile_interval": gecko_profile_interval,
             "gecko_profile_entries": gecko_profile_entries,
             "gecko_profile_dir": gecko_profile_dir,
             "gecko_profile_threads": gecko_profile_threads
         }
 
+        # profiles will be zipped up, but also stored locally on disk in a talos
+        # latest profiles folder, so they can easily be viewed later. The folder
+        # should be in ../testing/talos/talos/ so it is available to our webserver
+        if os.path.exists(self.browser_config['gecko_profiles_dir']):
+            mozfile.remove(self.browser_config['gecko_profiles_dir'])
+
     def option(self, name):
         return self.profiling_info["gecko_profile_" + name]
 
     def update_env(self, env):
         """
         update the given env to update some env vars if required.
         """
         if not self.test_config.get('gecko_profile_startup'):
@@ -195,15 +201,23 @@ class GeckoProfile(object):
                 except Exception:
                     LOG.exception(
                         "Failed to copy profile {0} as {1} to"
                         " archive {2}".format(profile_path,
                                               path_in_zip,
                                               self.profile_arcname)
                     )
 
+        # we also want to keep a readable copy of our profiles so
+        # that we can view them in the gecko profiler without unzipping
+        # (only when running locally, in production the zip is uploaded)
+        if self.browser_config['develop']:
+            mozfile.extract(self.profile_arcname, self.browser_config['gecko_profiles_dir'])
+            LOG.info("Profiles are also available in {0}"
+                     .format(self.browser_config['gecko_profiles_dir']))
+
     def clean(self):
         """
         Clean up temp folders created with the instance creation.
         """
         mozfile.remove(self.option('dir'))
         for symbol_path in self.symbol_paths.values():
             mozfile.remove(symbol_path)
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -140,16 +140,18 @@ def run_tests(config, browser_config):
 
     # fix paths to substitute
     # `os.path.dirname(os.path.abspath(__file__))` for ${talos}
     # https://bugzilla.mozilla.org/show_bug.cgi?id=705809
     browser_config['extensions'] = [utils.interpolate(i)
                                     for i in browser_config['extensions']]
     browser_config['bcontroller_config'] = \
         utils.interpolate(browser_config['bcontroller_config'])
+    browser_config['gecko_profiles_dir'] = \
+        utils.interpolate(browser_config['gecko_profiles_dir'])
 
     # normalize browser path to work across platforms
     browser_config['browser_path'] = \
         os.path.normpath(browser_config['browser_path'])
 
     binary = browser_config["browser_path"]
     version_info = mozversion.get_version(binary=binary)
     browser_config['browser_name'] = version_info['application_name']