mozilla-tests/universal_master_sqlite.cfg
author Philipp Kewisch <mozilla@kewis.ch>
Mon, 26 Feb 2018 13:12:57 +0100
changeset 15659 611002f8acd2e59db8e941a7e674ab4f52486c2c
parent 15391 5027326bdd46ce076fb54182edc1e5a30087fd04
permissions -rw-r--r--
Bug 1441123 - Update cak, fa, ms locales for Thunderbird. r=me

# vim: ft=python
import os
import sys
from copy import deepcopy

# import/reload dance so that reconfigs work
import buildbotcustom.misc
import buildbotcustom.status.queued_command
import master_common
import config
import thunderbird_config
import master_localconfig
reload(buildbotcustom.misc)
reload(buildbotcustom.status.queued_command)
reload(master_common)
reload(config)
reload(thunderbird_config)
reload(master_localconfig)

from master_common import BuildmasterConfig
c = BuildmasterConfig
for key, value in master_localconfig.BuildmasterConfig.iteritems():
    if key in c:
        c[key].extend(value)
    else:
        c[key] = value

# Create our QueueDir objects
# This is reloaded in buildbotcustom.misc
from mozilla_buildtools.queuedir import QueueDir
commandsQueue = QueueDir('commands', '%s/commands' % master_localconfig.QUEUEDIR)
from buildbotcustom.status.queued_command import QueuedCommandHandler
c['status'].append(QueuedCommandHandler(
    command=[sys.executable, os.path.join(os.path.dirname(buildbotcustom.__file__), 'bin', 'postrun.py'), '-c', os.path.abspath(os.path.join(os.curdir, 'postrun.cfg'))],
    queuedir=commandsQueue,
))

from config import BRANCHES, PLATFORMS, SUITES, BRANCH_UNITTEST_VARS, SLAVES, TRY_SLAVES, PROJECTS
from thunderbird_config import BRANCHES as THUNDERBIRD_BRANCHES
from thunderbird_config import PLATFORMS as THUNDERBIRD_PLATFORMS
from thunderbird_config import BRANCH_UNITTEST_VARS as THUNDERBIRD_BRANCH_UNITTEST_VARS
from thunderbird_config import SUITES as THUNDERBIRD_SUITES
from master_localconfig import ACTIVE_BRANCHES, ACTIVE_PLATFORMS, ACTIVE_PROJECTS, \
    ACTIVE_THUNDERBIRD_BRANCHES, ACTIVE_THUNDERBIRD_PLATFORMS, \
    ACTIVE_FX_SLAVE_PLATFORMS

import BuildSlaves
reload(BuildSlaves)
from BuildSlaves import SlavePasswords
from buildbot.buildslave import BuildSlave

# Handle active platforms - Firefox
all_slave_platforms = []
for p in ACTIVE_PLATFORMS.keys():
    ACTIVE_PLATFORMS[p] = deepcopy(PLATFORMS[p])
    # Handle active slave platforms
    if p in ACTIVE_FX_SLAVE_PLATFORMS:
        for sp in ACTIVE_PLATFORMS[p]['slave_platforms'][:]:
            if sp not in ACTIVE_FX_SLAVE_PLATFORMS[p]:
                ACTIVE_PLATFORMS[p]['slave_platforms'].remove(sp)
                if sp in ACTIVE_PLATFORMS[p]:
                    del ACTIVE_PLATFORMS[p][sp]
        if 'talos_slave_platforms' in ACTIVE_PLATFORMS[p]:
            for sp in ACTIVE_PLATFORMS[p]['talos_slave_platforms'][:]:
                if sp not in ACTIVE_FX_SLAVE_PLATFORMS[p]:
                    ACTIVE_PLATFORMS[p]['talos_slave_platforms'].remove(sp)
                    if sp in ACTIVE_PLATFORMS[p]:
                        del ACTIVE_PLATFORMS[p][sp]

    all_slave_platforms.extend(ACTIVE_PLATFORMS[p]['slave_platforms'])
    if 'talos_slave_platforms' in ACTIVE_PLATFORMS[p]:
        all_slave_platforms.extend(ACTIVE_PLATFORMS[p]['talos_slave_platforms'])

# Handle active platforms - Thunderbird
tb_all_slave_platforms = []
for p in ACTIVE_THUNDERBIRD_PLATFORMS.keys():
    #XXX we shouldn't have to filter these platforms
    if p in THUNDERBIRD_PLATFORMS:
        ACTIVE_THUNDERBIRD_PLATFORMS[p] = deepcopy(THUNDERBIRD_PLATFORMS[p])
        tb_all_slave_platforms.extend(THUNDERBIRD_PLATFORMS[p]['slave_platforms'])

# Remove non-active platform entries for projects
for p in PROJECTS:
    if 'platforms' in PROJECTS[p]:
        for project_slave_platform in PROJECTS[p]['platforms'].keys():
            if project_slave_platform not in all_slave_platforms:
                del PROJECTS[p]['platforms'][project_slave_platform]

for slave_platform, slaves in SLAVES.iteritems():
    # If this platform isn't active, skip it!
    if slave_platform not in all_slave_platforms + tb_all_slave_platforms:
        continue
    for name in sorted(slaves.keys()):
        c['slaves'].append(BuildSlave(name, SlavePasswords.get(slave_platform, "pass"), max_builds=1,
                                      properties=slaves[name]))

if 'try' in ACTIVE_BRANCHES:
    for slave_platform, slaves in TRY_SLAVES.iteritems():
        # If this platform isn't active, skip it!
        if slave_platform not in all_slave_platforms + tb_all_slave_platforms:
            continue
        for name in sorted(slaves.keys()):
            c['slaves'].append(BuildSlave(name, SlavePasswords.get(slave_platform, "pass"), max_builds=1,
                                          properties=slaves[name]))

from buildbotcustom.misc import generateTalosBranchObjects, generateProjectObjects, \
    mergeBuildObjects

buildObjects = {}

for branch in ACTIVE_BRANCHES:
    talosObjects = generateTalosBranchObjects(branch, BRANCHES[branch],
                                              ACTIVE_PLATFORMS, BRANCHES[branch].get('suites', SUITES),
                                              BRANCH_UNITTEST_VARS['platforms'])
    buildObjects = mergeBuildObjects(buildObjects, talosObjects)

for branch in ACTIVE_THUNDERBIRD_BRANCHES:
    talosObjects = generateTalosBranchObjects(branch, THUNDERBIRD_BRANCHES[branch],
                                              ACTIVE_THUNDERBIRD_PLATFORMS,
                                              THUNDERBIRD_BRANCHES[branch].get('suites', THUNDERBIRD_SUITES),
                                              THUNDERBIRD_BRANCH_UNITTEST_VARS['platforms'])
    buildObjects = mergeBuildObjects(buildObjects, talosObjects)

for project in ACTIVE_PROJECTS:
    projectObjects = generateProjectObjects(project, PROJECTS[project], SLAVES)
    buildObjects = mergeBuildObjects(buildObjects, projectObjects)

c['builders'].extend(buildObjects['builders'])
c['status'].extend(buildObjects['status'])
c['schedulers'].extend(buildObjects['schedulers'])
c['change_source'].extend(buildObjects['change_source'])

# Required for sendchange
from buildbot.changes.pb import PBChangeSource
c['change_source'].append(PBChangeSource())

c['db_url'] = "sqlite:///state.sqlite"

c['mergeRequests'] = buildbotcustom.misc.mergeRequests

import passwords
reload(passwords)
if hasattr(passwords, 'PULSE_PASSWORD'):
    pulseQueue = QueueDir('pulse', '%s/pulse' % master_localconfig.QUEUEDIR)
    # Send pulse messages
    import re
    import buildbotcustom.status.pulse
    reload(buildbotcustom.status.pulse)
    from buildbotcustom.status.pulse import PulseStatus
    c['status'].append(PulseStatus(
        pulseQueue,
        ignoreBuilders=[re.compile('fuzzer-.*')],
        send_logs=False,
    ))

# c is shorthand for BuildmasterConfig, so make sure they still refer to the
# same thing
assert c is BuildmasterConfig

# Check that all our builders have branch, platform and product set
for b in c['builders']:
    assert 'properties' in b, b
    assert 'branch' in b['properties'], b
    assert 'platform' in b['properties'], b
    assert 'product' in b['properties'], b

# Check that we don't have more too many  builders for any one slave
from collections import defaultdict
_buildersBySlave = defaultdict(int)
for b in c['builders']:
    for s in b['slavenames']:
        _buildersBySlave[s] += 1

for s, count in _buildersBySlave.iteritems():
    # There are around 12 other PB refs used between the master and slave that
    # aren't builders
    builderLimit = 4096 - 12
    pct_of_max = int(float(count) / float(builderLimit) * 100)
    if pct_of_max > 90:
        print "%s has %i builders; limit is %i; %d percent of max" % (s, count, builderLimit, pct_of_max)
    assert count <= builderLimit, "%s has %i builders; limit is %i" % (s, count, builderLimit)

del _buildersBySlave