woo_client.py
author Geoff Brown <gbrown@mozilla.com>
Thu, 14 Sep 2017 17:25:38 -0600
changeset 340 bab5e465940d4ad84ec86be9b6fad5e62dd954de
parent 312 f2af7588d7f242e470909107bdbc0ace5a850229
permissions -rw-r--r--
Bug 1399224 - Update whiteboard and comment on bugs with more than 200 intermittent failures in 30 days; r=jmaher

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

from collections import defaultdict

import requests


class TopBugs(object):

    def __init__(self, local_server_url, startday, endday, tree='trunk'):
        self.local_server_url = local_server_url
        self.startday = startday
        self.endday = endday
        self.tree = tree
        self.bugdata = {}

    def stats(self):
        print 'Getting orange data for %s to %s...' % (self.startday, self.endday)
        jdata = self.get_bybug()
        results = {}
        results['orangecount'] = sum([x['orangecount'] for x in jdata['oranges'].values() if x['testruns'] > 0])
        results['testruncount'] = sum([x['testruns'] for x in jdata['oranges'].values()])
        if results['testruncount'] > 0:
            results['orangefactor'] = float(results['orangecount']) / float(results['testruncount'])
            results['orangefactorstr'] = '%.1f' % results['orangefactor']
        else:
            results['orangefactor'] = None
            results['orangefactorstr'] = '?'
        return results

    def stats_by_bug(self):
        """Return the per-repository, per-platform and total failure counts for each bug seen.

        eg:
        {
            "1206327": {
                "total": 5,
                "per_repository": {
                    "fx-team": 2,
                    "mozilla-inbound": 3
                },
                "per_platform": {
                    "osx-10-10": 4,
                    "b2g-emu-ics": 1
                }
            },
            ...
        }
        """
        jdata = self.get_bybug()
        # We can't use collections.Counter since brasstacks is running Python 2.6.
        stats = defaultdict(lambda: {
            'total': 0,
            'per_repository': defaultdict(int),
            'per_platform': defaultdict(int),
        })
        for date in jdata['oranges'].values():
            for failure in date['oranges']:
                bug_id = failure['bug']
                stats[bug_id]['total'] += 1
                stats[bug_id]['per_repository'][failure['branch']] += 1
                stats[bug_id]['per_platform'][failure['platform']] += 1
        return stats

    def top_bugs(self):
        jdata = self.get_bybug()
        bug_counts = defaultdict(int)
        for date in jdata['oranges'].values():
            for orange in date['oranges']:
                bug_counts[orange['bug']] += 1
        bug_array = [(x, bug_counts[x]) for x in bug_counts.keys()]
        bug_array.sort(key=lambda x: x[1], reverse=True)
        if not bug_array:
            print "Bug orange data was:\n%s" % jdata['oranges']
            raise Exception("No bugs found in top_bugs!")
        return bug_array

    def bug_details(self, buglist):
        jdata = self.get_bugdata('/bugdetails?bugid=%s' % ','.join(buglist))
        return jdata['bugs']

    def get_json(self, url_path):
        url = self.local_server_url + url_path
        print '-> Fetching JSON from %s' % url
        session = requests.Session()
        # Use a custom HTTP adapter, so we can set a non-zero max_retries value.
        # See http://www.python-requests.org/en/latest/api/#requests.adapters.HTTPAdapter
        retrying_adapter = requests.adapters.HTTPAdapter(max_retries=3)
        session.mount("http://", retrying_adapter)
        session.mount("https://", retrying_adapter)
        resp = session.get(url, timeout=120, headers={
            'Accept': 'application/json',
            'User-Agent': 'orangefactor-mailer',
        })
        try:
            resp.raise_for_status()
        except requests.exceptions.HTTPError:
            print "HTTPError %s fetching %s: %s" % (resp.status_code, url, resp.text)
            raise
        return resp.json()

    def get_bugdata(self, url_path):
        print 'Getting bug data...'
        if url_path in self.bugdata:
            print '-> Found in cache: %s' % url_path
            return self.bugdata[url_path]
        jdata = self.get_json(url_path)
        self.bugdata[url_path] = jdata
        return jdata

    def get_bybug(self):
        return self.get_bugdata('/bybug?tree=%s&startday=%s&endday=%s' % (self.tree, self.startday, self.endday))