build/submit_telemetry_data.py
author Justin Wood <Callek@gmail.com>
Thu, 13 Sep 2018 02:32:05 +0000
changeset 436142 79d78bba6f6871cf37fe793e26aa1e5ae33d8125
parent 385817 6074db12d685655fe5692d59471b3c32cc967dc9
child 437475 ce05cf6d5e1990c93c5c0220ea9354b4b646786f
permissions -rw-r--r--
Bug 1486970 - Create revision and pushlog-id index routes for cron decision tasks. r=dustin,aki This hardcodes the push info in .taskgraph.yml for cron so that we don't break Chain Of Trust (signing) on other branches (uplifts) when we adjust scriptworker to start passing in this data to json-e. This allows us to incrementally roll this out without needing a flag day, and will be needed on all production branches. Differential Revision: https://phabricator.services.mozilla.com/D4501

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import errno
import logging
import os
import sys
import time

HERE = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(HERE, '..', 'third_party', 'python', 'requests'))
import requests


# Server to which to submit telemetry data
BUILD_TELEMETRY_SERVER = 'http://52.88.27.118/build-metrics-dev'


def submit_telemetry_data(statedir):

    # No data to work with anyway
    outgoing = os.path.join(statedir, 'telemetry', 'outgoing')
    if not os.path.isdir(outgoing):
        return 0

    submitted = os.path.join(statedir, 'telemetry', 'submitted')
    try:
        os.mkdir(submitted)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    session = requests.Session()
    for filename in os.listdir(outgoing):
        path = os.path.join(outgoing, filename)
        if os.path.isdir(path) or not path.endswith('.json'):
            continue
        with open(path, 'r') as f:
            data = f.read()
            try:
                r = session.post(BUILD_TELEMETRY_SERVER, data=data,
                                 headers={'Content-Type': 'application/json'})
            except Exception as e:
                logging.error('Exception posting to telemetry '
                              'server: %s' % str(e))
                break
            # TODO: some of these errors are likely not recoverable, as
            # written, we'll retry indefinitely
            if r.status_code != 200:
                logging.error('Error posting to telemetry: %s %s' %
                              (r.status_code, r.text))
                continue

        os.rename(os.path.join(outgoing, filename),
                  os.path.join(submitted, filename))

    session.close()

    # Discard submitted data that is >= 30 days old
    now = time.time()
    for filename in os.listdir(submitted):
        ctime = os.stat(os.path.join(submitted, filename)).st_ctime
        if now - ctime >= 60 * 60 * 24 * 30:
            os.remove(os.path.join(submitted, filename))

    return 0


if __name__ == '__main__':
    if len(sys.argv) != 2:
        print('usage: python submit_telemetry_data.py <statedir>')
        sys.exit(1)
    statedir = sys.argv[1]
    logging.basicConfig(filename=os.path.join(statedir, 'telemetry', 'telemetry.log'),
                        format='%(asctime)s %(message)s')
    sys.exit(submit_telemetry_data(statedir))