build/submit_telemetry_data.py
author Andrew Osmond <aosmond@mozilla.com>
Thu, 29 Nov 2018 14:38:28 -0500
changeset 449414 0861a85d13bb1664c019b76fc543d2679e39c99f
parent 445912 9a2f0d443bb1d94c38da120b790fcad6245b77d7
permissions -rw-r--r--
Bug 1510601 - Part 1. Move size increments into AnimationFrameBuffer::InsertInternal. r=tnikkel The size was originally incremented in AnimationFrameBuffer::Insert however if an animation was reset before we finished decoding, it would count some frames twice in the counter. Now we increment it inside InsertInternal, where AnimationFrameDiscardingQueue can make a more informed decision on whether the frame is a duplicate or not. Additionally we now fail explicitly when we insert more frames on subsequent decodes than the original decoders. This will help avoid getting out of sync with FrameAnimator. Differential Revision: https://phabricator.services.mozilla.com/D13464

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import print_function

import datetime
import json
import logging
import os
import sys

import requests
import voluptuous
import voluptuous.humanize

from mozbuild.telemetry import (
    schema as build_telemetry_schema,
    verify_statedir,
)

BUILD_TELEMETRY_URL = 'https://incoming.telemetry.mozilla.org/{endpoint}'
SUBMIT_ENDPOINT = 'submit/eng-workflow/build/1/{ping_uuid}'
STATUS_ENDPOINT = 'status'


def delete_expired_files(directory, days=30):
    '''Discards files in a directory older than a specified number
    of days
    '''
    now = datetime.datetime.now()
    for filename in os.listdir(directory):
        filepath = os.path.join(directory, filename)

        ctime = os.path.getctime(filepath)
        then = datetime.datetime.fromtimestamp(ctime)

        if (now - then) > datetime.timedelta(days=days):
            os.remove(filepath)

    return


def check_edge_server_status(session):
    '''Returns True if the Telemetry Edge Server
    is ready to accept data
    '''
    status_url = BUILD_TELEMETRY_URL.format(endpoint=STATUS_ENDPOINT)
    response = session.get(status_url)
    if response.status_code != 200:
        return False
    return True


def send_telemetry_ping(session, data, ping_uuid):
    '''Sends a single build telemetry ping to the
    edge server, returning the response object
    '''
    resource_url = SUBMIT_ENDPOINT.format(ping_uuid=str(ping_uuid))
    url = BUILD_TELEMETRY_URL.format(endpoint=resource_url)
    response = session.post(url, json=data)

    return response


def submit_telemetry_data(outgoing, submitted):
    '''Sends information about `./mach build` invocations to
    the Telemetry pipeline
    '''
    with requests.Session() as session:
        # Confirm the server is OK
        if not check_edge_server_status(session):
            logging.error('Error posting to telemetry: server status is not "200 OK"')
            return 1

        for filename in os.listdir(outgoing):
            path = os.path.join(outgoing, filename)

            if os.path.isdir(path) or not path.endswith('.json'):
                logging.info('skipping item {}'.format(path))
                continue

            ping_uuid = os.path.splitext(filename)[0]  # strip ".json" to get ping UUID

            try:
                with open(path, 'r') as f:
                    data = json.load(f)

                # Verify the data matches the schema
                voluptuous.humanize.validate_with_humanized_errors(
                    data, build_telemetry_schema
                )

                response = send_telemetry_ping(session, data, ping_uuid)
                if response.status_code != 200:
                    msg = 'response code {code} sending {uuid} to telemetry: {body}'.format(
                        body=response.content,
                        code=response.status_code,
                        uuid=ping_uuid,
                    )
                    logging.error(msg)
                    continue

                # Move from "outgoing" to "submitted"
                os.rename(os.path.join(outgoing, filename),
                          os.path.join(submitted, filename))

                logging.info('successfully posted {} to telemetry'.format(ping_uuid))

            except ValueError as ve:
                # ValueError is thrown if JSON cannot be decoded
                logging.exception('exception parsing JSON at %s: %s'
                                  % (path, str(ve)))
                os.remove(path)

            except voluptuous.Error as e:
                # Invalid is thrown if some data does not fit
                # the correct Schema
                logging.exception('invalid data found at %s: %s'
                                  % (path, e.message))
                os.remove(path)

            except Exception as e:
                logging.error('exception posting to telemetry '
                              'server: %s' % str(e))
                break

    delete_expired_files(submitted)

    return 0


if __name__ == '__main__':
    if len(sys.argv) != 2:
        print('usage: python submit_telemetry_data.py <statedir>')
        sys.exit(1)

    statedir = sys.argv[1]

    try:
        outgoing, submitted, telemetry_log = verify_statedir(statedir)

        # Configure logging
        logging.basicConfig(filename=telemetry_log,
                            format='%(asctime)s %(message)s',
                            level=logging.DEBUG)

        sys.exit(submit_telemetry_data(outgoing, submitted))

    except Exception as e:
        # Handle and print messages from `statedir` verification
        print(e.message)
        sys.exit(1)