merge default->production; tests passing production
authorAndrei Obreja <aobreja@mozilla.com>
Mon, 12 Jun 2017 13:50:40 +0300
branchproduction
changeset 4472 38893e6100c17f64f1991f63e1cd78d1bd46545d
parent 4470 9a1328ff060acb388ad9dc084d60877ce7382a28 (current diff)
parent 4471 bac9d342dedf9bd210a11d3e33489866aa037535 (diff)
child 4474 20b02deaf60ab824fac729ead22be76527627719
push id3576
push useraobreja@mozilla.com
push dateMon, 12 Jun 2017 10:51:00 +0000
merge default->production; tests passing
--- a/mozharness/base/python.py
+++ b/mozharness/base/python.py
@@ -539,205 +539,14 @@ class ResourceMonitoringMixin(object):
         log_usage('Total resource usage', duration, cpu_percent, cpu_times, io)
 
         for phase in rm.phases.keys():
             start_time, end_time = rm.phases[phase]
             cpu_percent, cpu_times, io = resources(phase)
             log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
 
 
-class InfluxRecordingMixin(object):
-    """Provides InfluxDB stat recording to scripts.
-
-    This class records stats to an InfluxDB server, if enabled. Stat recording
-    is enabled in a script by inheriting from this class, and adding an
-    influxdb_credentials line to the influx_credentials_file (usually oauth.txt
-    in automation).  This line should look something like:
-
-        influxdb_credentials = 'http://goldiewilson-onepointtwentyone-1.c.influxdb.com:8086/db/DBNAME/series?u=DBUSERNAME&p=DBPASSWORD'
-
-    Where DBNAME, DBUSERNAME, and DBPASSWORD correspond to the database name,
-    and user/pw credentials for recording to the database. The stats from
-    mozharness are recorded in the 'mozharness' table.
-    """
-
-    @PreScriptRun
-    def influxdb_recording_init(self):
-        self.recording = False
-        self.post = None
-        self.posturl = None
-        self.res_props = self.config.get('build_resources_path') % self.query_abs_dirs()
-        self.info("build_resources.json path: %s" % self.res_props)
-        if self.res_props:
-            self.rmtree(self.res_props)
-
-        try:
-            site_packages_path = self.query_python_site_packages_path()
-            if site_packages_path not in sys.path:
-                sys.path.append(site_packages_path)
-
-            import requests
-            self.post = requests.post
-
-            auth = os.path.join(os.getcwd(), self.config['influx_credentials_file'])
-            credentials = {}
-            execfile(auth, credentials)
-            self.posturl = credentials['influxdb_credentials']
-
-            self.recording = True
-        except Exception:
-            # The exact reason for failing to start stats doesn't really matter.
-            # If anything fails, we just won't record stats for this job.
-            self.warning("Unable to start influxdb recording: %s" %
-                         traceback.format_exc())
-
-    @PreScriptAction
-    def influxdb_recording_pre_action(self, action):
-        if not self.recording:
-            return
-
-        self.start_time = time.time()
-
-    @PostScriptAction
-    def influxdb_recording_post_action(self, action, success=None):
-        if not self.recording:
-            return
-
-        elapsed_time = time.time() - self.start_time
-
-        c = {}
-        p = {}
-        if self.buildbot_config:
-            c = self.buildbot_config.get('properties', {})
-        if self.buildbot_properties:
-            p = self.buildbot_properties
-        self.record_influx_stat([{
-            "points": [[
-                action,
-                elapsed_time,
-                c.get('buildername'),
-                c.get('product'),
-                c.get('platform'),
-                c.get('branch'),
-                c.get('slavename'),
-                c.get('revision'),
-                p.get('gaia_revision'),
-                c.get('buildid'),
-            ]],
-            "name": "mozharness",
-            "columns": [
-                "action",
-                "runtime",
-                "buildername",
-                "product",
-                "platform",
-                "branch",
-                "slavename",
-                "gecko_revision",
-                "gaia_revision",
-                "buildid",
-            ],
-        }])
-
-    def _get_resource_usage(self, res, name, iolen, cpulen):
-        c = {}
-        p = {}
-        if self.buildbot_config:
-            c = self.buildbot_config.get('properties', {})
-        if self.buildbot_properties:
-            p = self.buildbot_properties
-
-        data = [
-            # Build properties
-            c.get('buildername'),
-            c.get('product'),
-            c.get('platform'),
-            c.get('branch'),
-            c.get('slavename'),
-            c.get('revision'),
-            p.get('gaia_revision'),
-            c.get('buildid'),
-
-            # Mach step properties
-            name,
-            res.get('start'),
-            res.get('end'),
-            res.get('duration'),
-            res.get('cpu_percent'),
-        ]
-        # The io and cpu_times fields are arrays, though they aren't always
-        # present if a step completes before resource utilization is measured.
-        # We add the arrays if they exist, otherwise we just do an array of None
-        # to fill up the stat point.
-        data.extend(res.get('io', [None] * iolen))
-        data.extend(res.get('cpu_times', [None] * cpulen))
-        return data
-
-    @PostScriptAction('build')
-    def record_mach_stats(self, action, success=None):
-        if not self.recording:
-            return
-        if not os.path.exists(self.res_props):
-            self.info('No build_resources.json found, not logging stats')
-            return
-        with open(self.res_props) as fh:
-            resources = json.load(fh)
-            data = {
-                "points": [
-                ],
-                "name": "mach",
-                "columns": [
-                    # Build properties
-                    "buildername",
-                    "product",
-                    "platform",
-                    "branch",
-                    "slavename",
-                    "gecko_revision",
-                    "gaia_revision",
-                    "buildid",
-
-                    # Mach step properties
-                    "name",
-                    "start",
-                    "end",
-                    "duration",
-                    "cpu_percent",
-                ],
-            }
-            # The io and cpu_times fields aren't static - they may vary based
-            # on the specific platform being measured. Mach records the field
-            # names, which we use as the column names here.
-            data['columns'].extend(resources['io_fields'])
-            data['columns'].extend(resources['cpu_times_fields'])
-            iolen = len(resources['io_fields'])
-            cpulen = len(resources['cpu_times_fields'])
-
-            # The top-level data has the overall resource usage, which we record
-            # under the name 'TOTAL' to separate it from the individual tiers.
-            data['points'].append(self._get_resource_usage(resources, 'TOTAL', iolen, cpulen))
-
-            # Each tier also has the same resource stats as the top-level.
-            for tier in resources['tiers']:
-                data['points'].append(self._get_resource_usage(tier, tier['name'], iolen, cpulen))
-
-            self.record_influx_stat([data])
-
-    def record_influx_stat(self, json_data):
-        try:
-            r = self.post(self.posturl, data=json.dumps(json_data), timeout=5)
-            if r.status_code != 200:
-                self.warning("Failed to log stats. Return code = %i, stats = %s" % (r.status_code, json_data))
-
-                # Disable recording for the rest of this job. Even if it's just
-                # intermittent, we don't want to keep the build from progressing.
-                self.recording = False
-        except Exception, e:
-            self.warning('Failed to log stats. Exception = %s' % str(e))
-            self.recording = False
-
-
 # __main__ {{{1
 
 if __name__ == '__main__':
     '''TODO: unit tests.
     '''
     pass
--- a/mozharness/mozilla/building/buildbase.py
+++ b/mozharness/mozilla/building/buildbase.py
@@ -529,17 +529,17 @@ def generate_build_ID():
 
 
 def generate_build_UID():
     return uuid.uuid4().hex
 
 
 class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
                   SigningMixin, VirtualenvMixin, MercurialScript,
-                  TransferMixin, InfluxRecordingMixin):
+                  TransferMixin):
     def __init__(self, **kwargs):
         # objdir is referenced in _query_abs_dirs() so let's make sure we
         # have that attribute before calling BaseScript.__init__
         self.objdir = None
         super(BuildScript, self).__init__(**kwargs)
         # epoch is only here to represent the start of the buildbot build
         # that this mozharn script came from. until I can grab bbot's
         # status.build.gettime()[0] this will have to do as a rough estimate