author Drew Willcoxon <>
Mon, 29 Oct 2018 16:13:49 +0000
changeset 443341 0cd201908c7d8d409478c5b95a66fb3ab42378c8
parent 437475 ce05cf6d5e1990c93c5c0220ea9354b4b646786f
child 445884 12c35ad7e844ec1d1c555ef57acac71e5c97d656
permissions -rw-r--r--
Bug 1498023 - Search shortcuts should hide all one-off UI when typed r=mak,Mardak * Disable or enable the one-offs per each new search based on whether the first char is "@". The patch does this in `onResultsAdded`, where other per-search initialization happens. * Remove the `disableOneOffButtons` option from the urlbar `search` method. It's not necessary anymore now that one-offs are automatically hidden for the only caller that uses this option (new tab with the "@engine" tiles). * Make the `oneOffSearchesEnabled` getter return the actual status of the one-off UI instead of relying on an `_oneOffSearchesEnabled` property. Differential Revision:

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at

from __future__ import print_function

import datetime
import json
import logging
import os
import sys

HERE = os.path.abspath(os.path.dirname(__file__))
PYTHIRDPARTY = os.path.join(HERE, '..', 'third_party', 'python')

# Add some required files to $PATH to ensure they are available
sys.path.append(os.path.join(HERE, '..', 'python', 'mozbuild', 'mozbuild'))
sys.path.append(os.path.join(PYTHIRDPARTY, 'requests'))
sys.path.append(os.path.join(PYTHIRDPARTY, 'voluptuous'))

import requests
import voluptuous
import voluptuous.humanize

from mozbuild.telemetry import schema as build_telemetry_schema

BUILD_TELEMETRY_URL = '{endpoint}'
SUBMIT_ENDPOINT = 'submit/eng-workflow/build/1/{ping_uuid}'

def delete_expired_files(directory, days=30):
    '''Discards files in a directory older than a specified number
    of days
    now =
    for filename in os.listdir(directory):
        filepath = os.path.join(directory, filename)

        ctime = os.path.getctime(filepath)
        then = datetime.datetime.fromtimestamp(ctime)

        if (now - then) > datetime.timedelta(days=days):


def check_edge_server_status(session):
    '''Returns True if the Telemetry Edge Server
    is ready to accept data
    status_url = BUILD_TELEMETRY_URL.format(endpoint=STATUS_ENDPOINT)
    response = session.get(status_url)
    if response.status_code != 200:
        return False
    return True

def send_telemetry_ping(session, data, ping_uuid):
    '''Sends a single build telemetry ping to the
    edge server, returning the response object
    resource_url = SUBMIT_ENDPOINT.format(ping_uuid=str(ping_uuid))
    url = BUILD_TELEMETRY_URL.format(endpoint=resource_url)
    response =, json=data)

    return response

def submit_telemetry_data(outgoing, submitted):
    '''Sends information about `./mach build` invocations to
    the Telemetry pipeline
    with requests.Session() as session:
        # Confirm the server is OK
        if not check_edge_server_status(session):
            logging.error('Error posting to telemetry: server status is not "200 OK"')
            return 1

        for filename in os.listdir(outgoing):
            path = os.path.join(outgoing, filename)

            if os.path.isdir(path) or not path.endswith('.json'):
      'skipping item {}'.format(path))

            ping_uuid = os.path.splitext(filename)[0]  # strip ".json" to get ping UUID

                with open(path, 'r') as f:
                    data = json.load(f)

                # Verify the data matches the schema
                    data, build_telemetry_schema

                response = send_telemetry_ping(session, data, ping_uuid)
                if response.status_code != 200:
                    msg = 'response code {code} sending {uuid} to telemetry: {body}'.format(

                # Move from "outgoing" to "submitted"
                os.rename(os.path.join(outgoing, filename),
                          os.path.join(submitted, filename))

      'successfully posted {} to telemetry'.format(ping_uuid))

            except ValueError as ve:
                # ValueError is thrown if JSON cannot be decoded
                logging.exception('exception parsing JSON at %s: %s'
                                  % (path, str(ve)))

            except voluptuous.Error as e:
                # Invalid is thrown if some data does not fit
                # the correct Schema
                logging.exception('invalid data found at %s: %s'
                                  % (path, e.message))

            except Exception as e:
                logging.error('exception posting to telemetry '
                              'server: %s' % str(e))


    return 0

def verify_statedir(statedir):
    '''Verifies the statedir is structured according to the assumptions of
    this script

    Requires presence of the following directories; will raise if absent:
    - statedir/telemetry
    - statedir/telemetry/outgoing

    Creates the following directories and files if absent (first submission):
    - statedir/telemetry/submitted

    telemetry_dir = os.path.join(statedir, 'telemetry')
    outgoing = os.path.join(telemetry_dir, 'outgoing')
    submitted = os.path.join(telemetry_dir, 'submitted')
    telemetry_log = os.path.join(telemetry_dir, 'telemetry.log')

    if not os.path.isdir(telemetry_dir):
        raise Exception('{} does not exist'.format(telemetry_dir))

    if not os.path.isdir(outgoing):
        raise Exception('{} does not exist'.format(outgoing))

    if not os.path.isdir(submitted):

    return outgoing, submitted, telemetry_log

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print('usage: python <statedir>')

    statedir = sys.argv[1]

        outgoing, submitted, telemetry_log = verify_statedir(statedir)

        # Configure logging
                            format='%(asctime)s %(message)s',

        sys.exit(submit_telemetry_data(outgoing, submitted))

    except Exception as e:
        # Handle and print messages from `statedir` verification