Bug 1315041 - Record environment state in perfherder options; r=wlach
authorGregory Szorc <gps@mozilla.com>
Thu, 03 Nov 2016 14:41:58 -0700
changeset 321102 950d659062002ae672e70f820a2e99d76582f9ae
parent 321101 de69c6acf5c51d239d4ef93657056117327653b8
child 321103 3d200aa1872f6cdbe8191715cd8053006a99812a
push id30919
push userphilringnalda@gmail.com
push dateSat, 05 Nov 2016 20:28:20 +0000
treeherdermozilla-central@572249b2ffb6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerswlach
bugs1315041
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1315041 - Record environment state in perfherder options; r=wlach Currently, perfherder data containing resource usage goes into the same bucket regardless of the execution environment. This means results for buildbot, taskcluster, and for different AWS instance types are all mixed together, making the data noisy. This commit adds extraOptions values to the perfherder data to identify: * buildbot vs taskcluster * the TC instance type Data will go to separate buckets and should be much more consistent. MozReview-Commit-ID: A0zSkdVI6ZM
testing/mozharness/mozharness/base/python.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -3,29 +3,31 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 '''Python usage, esp. virtualenv.
 '''
 
 import distutils.version
+import errno
 import os
 import subprocess
 import sys
 import json
 import socket
 import traceback
 import urlparse
 
 import mozharness
 from mozharness.base.script import (
     PostScriptAction,
     PostScriptRun,
     PreScriptAction,
+    ScriptMixin,
 )
 from mozharness.base.errors import VirtualenvErrorList
 from mozharness.base.log import WARNING, FATAL
 from mozharness.mozilla.proxxy import Proxxy
 
 external_tools_path = os.path.join(
     os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
     'external_tools',
@@ -488,17 +490,53 @@ class VirtualenvMixin(object):
 
     def activate_virtualenv(self):
         """Import the virtualenv's packages into this Python interpreter."""
         bin_dir = os.path.dirname(self.query_python_path())
         activate = os.path.join(bin_dir, 'activate_this.py')
         execfile(activate, dict(__file__=activate))
 
 
-class ResourceMonitoringMixin(object):
+# This is (sadly) a mixin for logging methods.
+class PerfherderResourceOptionsMixin(ScriptMixin):
+    def perfherder_resource_options(self):
+        """Obtain a list of extraOptions values to identify the env."""
+        opts = []
+
+        if 'TASKCLUSTER_INSTANCE_TYPE' in os.environ:
+            # Include the instance type so results can be grouped.
+            opts.append('taskcluster-%s' % os.environ['TASKCLUSTER_INSTANCE_TYPE'])
+        else:
+            # We assume !taskcluster => buildbot.
+            instance = 'unknown'
+
+            # Try to load EC2 instance type from metadata file. This file
+            # may not exist in many scenarios (including when inside a chroot).
+            # So treat it as optional.
+            # TODO support Windows.
+            try:
+                # This file should exist on Linux in EC2.
+                with open('/etc/instance_metadata.json', 'rb') as fh:
+                    im = json.load(fh)
+                    instance = im['aws_instance_type'].encode('ascii')
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                self.info('instance_metadata.json not found; unable to '
+                          'determine instance type')
+            except Exception:
+                self.warning('error reading instance_metadata: %s' %
+                             traceback.format_exc())
+
+            opts.append('buildbot-%s' % instance)
+
+        return opts
+
+
+class ResourceMonitoringMixin(PerfherderResourceOptionsMixin):
     """Provides resource monitoring capabilities to scripts.
 
     When this class is in the inheritance chain, resource usage stats of the
     executing script will be recorded.
 
     This class requires the VirtualenvMixin in order to install a package used
     for recording resource usage.
 
@@ -656,17 +694,17 @@ class ResourceMonitoringMixin(object):
                 {'name': 'io_write_bytes', 'value': io.write_bytes},
                 {'name': 'io.read_bytes', 'value': io.read_bytes},
                 {'name': 'io_write_time', 'value': io.write_time},
                 {'name': 'io_read_time', 'value': io.read_time},
             ])
 
             suites.append({
                 'name': '%s.overall' % perfherder_name,
-                'extraOptions': perfherder_options,
+                'extraOptions': perfherder_options + self.perfherder_resource_options(),
                 'subtests': overall,
 
             })
 
             for phase in rm.phases.keys():
                 phase_duration = rm.phases[phase][1] - rm.phases[phase][0]
                 subtests = [
                     {
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -45,17 +45,20 @@ from mozharness.mozilla.purge import Pur
 from mozharness.mozilla.mock import MockMixin
 from mozharness.mozilla.secrets import SecretsMixin
 from mozharness.mozilla.signing import SigningMixin
 from mozharness.mozilla.mock import ERROR_MSGS as MOCK_ERROR_MSGS
 from mozharness.mozilla.testing.errors import TinderBoxPrintRe
 from mozharness.mozilla.testing.unittest import tbox_print_summary
 from mozharness.mozilla.updates.balrog import BalrogMixin
 from mozharness.mozilla.taskcluster_helper import Taskcluster
-from mozharness.base.python import VirtualenvMixin
+from mozharness.base.python import (
+    PerfherderResourceOptionsMixin,
+    VirtualenvMixin,
+)
 
 AUTOMATION_EXIT_CODES = EXIT_STATUS_DICT.values()
 AUTOMATION_EXIT_CODES.sort()
 
 MISSING_CFG_KEY_MSG = "The key '%s' could not be determined \
 Please add this to your config."
 
 ERROR_MSGS = {
@@ -582,17 +585,17 @@ def generate_build_ID():
 
 
 def generate_build_UID():
     return uuid.uuid4().hex
 
 
 class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
                   SigningMixin, VirtualenvMixin, MercurialScript,
-                  SecretsMixin):
+                  SecretsMixin, PerfherderResourceOptionsMixin):
     def __init__(self, **kwargs):
         # objdir is referenced in _query_abs_dirs() so let's make sure we
         # have that attribute before calling BaseScript.__init__
         self.objdir = None
         super(BuildScript, self).__init__(**kwargs)
         # epoch is only here to represent the start of the buildbot build
         # that this mozharn script came from. until I can grab bbot's
         # status.build.gettime()[0] this will have to do as a rough estimate
@@ -1858,16 +1861,17 @@ or run without that action (ie: --no-{ac
 
         if 'duration' not in resources:
             self.info('resource usage lacks duration; ignoring')
             return None
 
         data = {
             'name': 'build times',
             'value': resources['duration'],
+            'extraOptions': self.perfherder_resource_options(),
             'subtests': [],
         }
 
         for phase in resources['phases']:
             if 'duration' not in phase:
                 continue
             data['subtests'].append({
                 'name': phase['name'],