thunderbird/master.cfg
author Alice Nodelman <alice@mozilla.com>
Thu, 10 Sep 2009 09:14:36 -0700
changeset 1489 e095d2e1a3447c87a62c1f18d88615ae9483b8b1
parent 1486 46b7df93aaddd134066c3faee1c25d5637c43a24
child 1495 6e2a5c72b29f3e5a931a856bdc85ad73fd05c434
permissions -rw-r--r--
bug 514070 (max dirty profile testing) patch for production/try p=anodelman, r=catlee

# -*- python -*-
# ex: set syntax=python:

# Shorthand
c = BuildmasterConfig = {}

####### BUILDSLAVES

import BuildSlaves
reload(BuildSlaves)
c['slaves'] = BuildSlaves.SlaveList

# 'slavePortnum' defines the TCP port to listen on. This must match the value
# configured into the buildslaves (with their --master option)

c['slavePortnum'] = 9010

def getConfig(branch, value, default=None):
    return branch.get(value, DEFAULTS.get(value, default))

# most of the config is in an external file
import config
reload(config)
from config import *

import os
debug = os.environ.get('DEBUG')

if debug:
    import sys

from buildbot import locks
from buildbot.process import factory
from buildbot.scheduler import Scheduler, Nightly, Periodic, Triggerable, AnyBranchScheduler
from buildbot.status.tinderbox import TinderboxMailNotifier
from buildbot.status.words import IRC
import buildbot.steps.source
reload(buildbot.steps.source)
from buildbot.steps.source import Mercurial

from buildbot.steps.shell import Compile, ShellCommand, WithProperties, TreeSize

import buildbotcustom.changes.hgpoller
import buildbotcustom.steps.misc
import buildbotcustom.steps.test
import buildbotcustom.steps.transfer
import buildbotcustom.steps.updates
import buildbotcustom.status.tinderbox
reload(buildbotcustom.changes.hgpoller)
reload(buildbotcustom.steps.misc)
reload(buildbotcustom.steps.test)
reload(buildbotcustom.steps.transfer)
reload(buildbotcustom.steps.updates)
reload(buildbotcustom.status.tinderbox)

from buildbotcustom.steps.test import AliveTest, CompareBloatLogs, \
  CompareLeakLogs, Codesighs
from buildbotcustom.steps.misc import SetMozillaBuildProperties, GetHgRevision, MozillaClobberer
from buildbotcustom.steps.transfer import MozillaStageUpload
from buildbotcustom.steps.updates import CreateCompleteUpdateSnippet

import buildbotcustom.process.factory
reload(buildbotcustom.process.factory)
from buildbotcustom.process.factory import CCUnittestBuildFactory, CCNightlyRepackFactory

# l10n parallelization logic
import buildbotcustom.log
import buildbotcustom.l10n
reload(buildbotcustom.log)
reload(buildbotcustom.l10n)
from buildbotcustom.l10n import NightlyL10n, Scheduler as SchedulerL10n

import mozillaleak
reload(mozillaleak)
from mozillaleak import addLeakTestSteps

import mozillabuild
reload(mozillabuild)
from mozillabuild import *

c['schedulers'] = []
c['builders'] = []
c['status'] = []

####### L10n Scheduler with multiple dispatchers

# for debugging purposes of the repack on locale change system
buildbotcustom.log.init(
    scheduler = buildbotcustom.log.DEBUG,
    dispatcher = buildbotcustom.log.DEBUG
)

s = SchedulerL10n("l10n", "l10nbuilds.ini")
c['schedulers'].append(s)

####### Release Automation

import release_master
reload(release_master)

c['builders'].extend(release_master.builders)
c['schedulers'].extend(release_master.schedulers)
c['change_source'].extend(release_master.change_source)
c['status'].extend(release_master.status)

####### SOURCES

c['change_source'] = []

setupHGPollersFromBranches(DEFAULTS, BRANCHES, c['change_source'], 'comm-central')

for name in BRANCHES.keys():
    # shorthand 
    branch = BRANCHES[name]
    build_factory = getConfig(DEFAULTS, branch, 'factory')
    master_branch = getConfig(DEFAULTS, branch, 'master_branch')
    mozilla_branch = getConfig(DEFAULTS, branch, 'mozilla_central_branch')
    l10n = getConfig(DEFAULTS, branch,'l10n')
    builders = []
    nightlyBuilders = []
    localeBuilders = []
    allBuilders = []
    
    if debug:
        print >> sys.stderr, "Branch: %s, Factory %s, L10n=%s" % (name, build_factory,l10n)

    nightly = branch.get('nightly') != False
    
    product = branch.get('product', PRODUCT)
    branch['product'] = product
    appname=branch.get('appname', MOZ_APP_NAME)
    branch['appname']= appname
    brandname=getConfig(DEFAULTS, branch, 'brand_name')
    product_name=getConfig(DEFAULTS, branch, 'product_name')
    branch['brand_name'] = brandname

    branch_name = getConfig(DEFAULTS, branch, 'branch_name')
    
    mozconfig=branch.get('mozconfig','mozconfig')
    aus2_host = branch.get('aus2_host', AUS2_HOST)
    aus2_user = branch.get('aus2_user', AUS2_USER)
    download_base_url = branch.get('download_base_url', DOWNLOAD_BASE_URL)
    stage_base_path = getConfig(DEFAULTS, branch, 'stage_base_path')
    builder_type = getConfig(DEFAULTS, branch, 'builder_type')
    
    # generate a list of builders, nightly builders (names must be different)
    # for easy access
    for platform in branch['platforms'].keys():
        platform_l10n = branch['platforms'][platform].get('l10n', l10n)
        if debug: 
            print >> sys.stderr, "Branch: %s Nightly: %s" % (name, nightly)
        
        builders.append('%s %s' % (branch['platforms'][platform]['base_name'], builder_type))
        
        # Not everybody wants nightlies
        if nightly:
            nightlyBuilders.append('%s nightly' % \
                                    branch['platforms'][platform]['base_name'])
        # Not everybody wants l10n repacks
        if platform_l10n:
            l10n_builder_name = "%s %s l10n" % (product_name, branch['platforms'][platform]['base_name'])
            localeBuilders.append("%s" % (l10n_builder_name))
            localeBuilders.append("%s build" % (l10n_builder_name))

    allBuilders.extend(builders)
    allBuilders.extend(nightlyBuilders)

    error_parser = "unix"
    if builder_type == "check":
        error_parser = "unittest"

    # Currently, each branch goes to a different tree
    # If this changes in the future this may have to be
    # moved out of the loop
    tinderbox_tree = getConfig(DEFAULTS, branch, 'tinderbox_tree')
    c['status'].append(TinderboxMailNotifier(
        fromaddr="gozer@mozillamessaging.com",
        tree=tinderbox_tree,
        extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org", ],
        relayhost="mail.build.mozilla.org",
        builders=allBuilders,
        logCompression="bzip2",
        errorparser = error_parser, 
    ))
    if l10n:
        # This notifies all l10n related build objects to Mozilla-l10n
        c['status'].append(TinderboxMailNotifier(
            fromaddr="gozer@mozillamessaging.com",
            tree="Mozilla-l10n",
            extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
            relayhost="mail.build.mozilla.org",
            logCompression="bzip2",
            builders=localeBuilders,
            binaryURL="%s/nightly/latest-%s-l10n/" % (download_base_url, branch_name),
        ))
        # We only want the builds from the specified builders
        # since their builds have a build property called "locale"
        c['status'].append(TinderboxMailNotifier(
            fromaddr="gozer@mozillamessaging.com",
            tree=WithProperties("Mozilla-l10n-%(locale)s"),
            extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
            relayhost="mail.build.mozilla.org",
            logCompression="bzip2",
            builders=localeBuilders,
            binaryURL="%s/nightly/latest-%s-l10n/" % (download_base_url, branch_name),
        ))
        

    if DEFAULTS['irc']: 
        c['status'].append(IRC(
            host='irc.mozilla.org',
            nick=branch['irc_nick'],
            channels=branch['irc_channels'],
            categories=[name],
        ))
    
    # Hg branch defaults to the branch name, or otherwise specified by hg_branch
    hg_branch = branch.get('hg_branch',name)

    # schedulers
    # this one gets triggered by the HG Poller
    add_poll_branches = getConfig(DEFAULTS, branch, 'add_poll_branches')
    c['schedulers'].append(AnyBranchScheduler(
        name=name,
        branches=[hg_branch, mozilla_branch] + add_poll_branches,
        treeStableTimer=3*60,
#        treeStableTimer=60*60*12, #XXX: Disable
        builderNames=builders
    ))
    if debug:
        print >> sys.stderr, "AnyBranchScheduler for %s watches %s" % (name, [hg_branch, mozilla_branch] + add_poll_branches)
        print >> sys.stderr, "hg_branch=%s for %s" % (hg_branch, name)
    c['schedulers'].append(Nightly(
        name='%s nightly' % name,
        branch=hg_branch,
        hour=[3],
        builderNames=nightlyBuilders
    ))
    if debug:
        print >> sys.stderr, "Periodic(name=%s buildernames=%s branch=%s BHuildTimer=%s)" %(name, builders, hg_branch, getConfig(DEFAULTS, branch, 'period'))
    c['schedulers'].append(Periodic(
        name='%s periodic' % name,
        builderNames=builders,
        branch=hg_branch,
        periodicBuildTimer=getConfig(DEFAULTS, branch, 'period'),
    ))
    if debug:
        print >> sys.stderr, "L10n=%s for Name: %s" % (l10n, name)
    if l10n:
        #XXX: Bug, I get 2 of these, because of Calendar. Fixme
        hg_all_locales_poller = HgAllLocalesPoller(hgURL = HGURL,
                            repositoryIndex =  getConfig(DEFAULTS, branch, 'l10n_repo'),
                            pollInterval = 5*60)
        hg_all_locales_poller.parallelRequests = 1
        c['change_source'].append(hg_all_locales_poller)


    for platform in branch['platforms'].keys():
        # no need to have a whole new directory just for logs.
        # let's put these in their non-debug counterpart
        # translates to, eg:
        #  /home/ftp/pub/thunderbird/tinderbox-builds/comm-central-linux

        #XXX: Kludge
        realPlatform = platform.replace('-debug', '').replace('64', '').replace('-shark', '')
        realPlatform = realPlatform.replace('-10.5','')

        logUploadDir = 'tinderbox-builds/%s-%s/' % (name, platform)

        lockname = 'lock-%s-%s-dep' % (name, platform)
        lock = locks.SlaveLock(lockname)

        # shorthand
        pf = branch['platforms'][platform]

        # Milestone controls where things get uploaded on stage
        milestone = pf.get('milestone',  branch.get('milestone', DEFAULTS.get('milestone', name)))
        if debug:
            print "Branch: %s Platform: %s Milestone: %s" % (name, platform, milestone)

        codesighs = pf.get('codesighs', getConfig(DEFAULTS, branch, 'codesighs'))
        upload_stage = pf.get('upload_stage', getConfig(DEFAULTS, branch, 'upload_stage'))
        l10n = pf.get('l10n', getConfig(DEFAULTS, branch, 'l10n'))
        cvsroot = pf.get('cvsroot', getConfig(DEFAULTS, branch, 'cvsroot'))
        
        upload_glob = None
        if branch.get('upload_glob'):
            upload_glob="%s/%s" % (pf['platform_objdir'], branch['upload_glob'])
        if pf.get('upload_glob'):
            upload_glob="%s/%s" % (pf['platform_objdir'], pf['upload_glob'])

        #Clobber support
        clobber_url = getConfig(DEFAULTS, branch, 'clobber_url')
        build_tools_repo = getConfig(DEFAULTS, branch, 'build_tools_repo')

        if build_factory != 'build':
            client_py_args = getConfig(DEFAULTS, branch, 'client_py_args')
            if debug:
                print >> sys.stderr, "CCUnittestBuildFactory!!"
                print >> sys.stderr, "Builders are: %s" % builders
            unittest_factory = CCUnittestBuildFactory(
       		    branchName=branch_name,
                    platform=platform,
                    config_repo_path='build/buildbot-configs',
                    config_dir=CONFIG_SUBDIR,
                    objdir='objdir',
                    productName=MOZ_APP_NAME,
                    brandName='Thunderbird',
                    mochitest_leak_threshold=None,
                    mochichrome_leak_threshold=None,
                    mochibrowser_leak_threshold=None,
                    hgHost='hg.mozilla.org',
                    repoPath='comm-central',
                    mozRepoPath=mozilla_branch,
                    buildToolsRepoPath=build_tools_repo,
                    buildSpace=None,
                    clobberURL=clobber_url,
                    clobberTime=30*24*7, #Hours before clobber
                    buildsBeforeReboot=None,
                    exec_reftest_suites=False,
                    exec_mochi_suites=False,
                )
            
            builder = {
                'name': '%s %s' % (pf['base_name'], builder_type),
                'slavenames': pf['slaves'],
                'builddir': "%s-%s-%s" % (platform, branch_name, builder_type),
                'factory': unittest_factory,
                'category': name,
            }
            c['builders'].append(builder)
            
            continue

        mozilla2_dep_factory = factory.BuildFactory()
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', WithProperties('Building on: %(slavename)s')],
            env=pf['env']
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties('s: %(slavename)s')]
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command="rm -rfv %s/dist/thunderbird-* %s/dist/install/sea/*.exe " %
                     (pf['platform_objdir'], pf['platform_objdir']),
            env=pf['env'],
            description=['deleting', 'old', 'package'],
            descriptionDone=['delete', 'old', 'package']
        ))
        
        mozilla2_dep_factory.addStep(ShellCommand,
         command=['rm', '-rf', 'tools'],
         description=['clobber', 'build tools'],
         workdir='.'
        )
        mozilla2_dep_factory.addStep(ShellCommand,
         command=['bash', '-c',
          'if [ ! -d tools ]; then hg clone %s%s; fi' % (HGURL, build_tools_repo)],
         description=['clone', 'build tools'],
         workdir='.',
         flunkOnFailure=False,
         haltOnFailure=False,
        )

        clobberer_workdir = 'build/%s' % pf['platform_objdir']
        
        #XXX: This is a little brittle, assumes only objdir with ppc in them are 2-leveled     
        import os.path
        if platform.startswith("macosx") and clobberer_workdir.find('ppc') != -1:
            clobberer_workdir = os.path.dirname(clobberer_workdir)
        if platform.startswith("macosx") and clobberer_workdir.find('i386') != -1:
            clobberer_workdir =  os.path.dirname(clobberer_workdir)
        
        mozilla2_dep_factory.addStep(MozillaClobberer,
            branch=branch_name,
            clobber_url=clobber_url,
            clobberTime = 30*24*7, #Hours before clobber
            clobberer_path='../../tools/clobberer/clobberer.py',
            workdir=clobberer_workdir,
            flunkOnFailure=False,
            haltOnFailure=False,
            timeout=3600, # One hour, because Windows is slow
        )
        #Clobber support end


        mozilla2_dep_factory.addStep(Mercurial(
            branchType='inrepo',
            mode='update',
            repourl="%s%s" % (HGURL, hg_branch),
            alwaysUseLatest=True,
            timeout=60*60,
            retry=[60,3],
            clobberOnBranchChange=False,
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, hg_branch)
        changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['python', 'client.py' ] + branch['client_py_args'] + ['--cvsroot', cvsroot, 'checkout'],
            description=['running', 'client.py', 'checkout'],
            descriptionDone=['python', 'client.py', 'checkout'],
            haltOnFailure=True,
            timeout=60*60,
        ))
        mozilla2_dep_factory.addStep(GetHgRevision(
            workdir='build/mozilla'
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, mozilla_branch)

        link_tag = 'moz'
        if mozilla_branch == 'mozilla-central':
            link_tag = 'm-c'
        changesetLink += '/%(hg_revision)s title="Built from Mozilla revision %(hg_revision)s">' + link_tag + ':%(hg_revision)s</a>'

        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['rm', '-rfv', 'configs'],
            description=['removing', 'configs'],
            descriptionDone=['remove', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['hg', 'clone', CONFIG_REPO_URL, 'configs'],
            description=['checking', 'out', 'configs'],
            descriptionDone=['checkout', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            # cp configs/thunderbird/$platform/mozconfig .mozconfig
            command=['cp', 'configs/%s/%s/%s' % (CONFIG_SUBDIR,
                                                 platform,
                                                 mozconfig
                                                 ),
                     '.mozconfig'],
            description=['copying', 'mozconfig'],
            descriptionDone=['copy', 'mozconfig'],
            haltOnFailure=True
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['cat', '.mozconfig'],
        ))

        buildcmd = 'build'
        mozilla2_dep_factory.addStep(Compile(
            command=['make', '-f', 'client.mk', buildcmd],
            env=pf['env'],
            haltOnFailure=True,
            timeout=60*60
        ))

        if branch.get('leak'):
            addLeakTestSteps(mozilla2_dep_factory, branch, pf, platform)

        if platform.find('debug') == -1:
            if branch['package']:
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['make', 'package'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=True
                    ))
                    if platform.startswith("win32"):
                        mozilla2_dep_factory.addStep(ShellCommand(
                            command=['make', 'installer'],
                            workdir='build/%s' % pf['platform_objdir'],
                            haltOnFailure=True
                        ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['make', 'package-compare'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=False
                    ))
            mozilla2_dep_factory.addStep(SetMozillaBuildProperties(
                objdir='build/%s/mozilla' % pf['platform_objdir']
            ))
            if upload_stage:
                mozilla2_dep_factory.addStep(MozillaStageUpload(
                    objdir='%s/mozilla' % pf['platform_objdir'],
                    username=branch.get('stage_username',STAGE_USERNAME),
                    milestone=milestone,
                    remoteHost=branch.get('stage_server',STAGE_SERVER),
                    remoteBasePath=stage_base_path,
                    packageGlob=upload_glob,
                    platform=realPlatform,
                    group=branch.get('stage_group',STAGE_GROUP),
                    sshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    releaseToLatest=False,
                    releaseToDated=False,
                    releaseToTinderboxBuilds=True,
                    tinderboxBuildsDir='%s-%s' % (milestone, platform),
                    dependToDated=True
                ))
            if codesighs:
                if platform.find('win32') == -1 and platform.find('linux64') == -1:
                    # Codesighs
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['make'],
                        workdir='build/%s/mozilla/tools/codesighs' % pf['platform_objdir']
                    ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['wget', '-O', 'codesize-auto-old.log',
                         'http://%s/pub/mozilla.org/thunderbird/%s/codesize-auto.log' %\
                          (branch.get('stage_server',STAGE_SERVER), logUploadDir)],
                        env=pf['env']
                    ))
                    mozilla2_dep_factory.addStep(Codesighs(
                        objdir='../%s/mozilla' % pf['platform_objdir'],
                        platform=realPlatform,
                        workdir='build/mozilla',
                        env=pf['env']
                    ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['cat', 'codesize-auto-diff.log']
                    ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['scp', '-o', 'User=%s' % branch.get('stage_username',STAGE_USERNAME),
                         '-o', 'IdentityFile=~/.ssh/%s' % branch.get('stage_ssh_key',STAGE_SSH_KEY),
                         'codesize-auto.log',
                         '%s:%s/%s' % (branch.get('stage_server',STAGE_SERVER), stage_base_path, logUploadDir)]
                    ))

        mozilla2_dep_builder = {
            'name': '%s %s' % (pf['base_name'], builder_type),
            'slavenames': pf['slaves'],
            'builddir': '%s-%s' % (name, platform),
            'factory': mozilla2_dep_factory,
            'category': name,
            'locks' : [lock],
        }
        c['builders'].append(mozilla2_dep_builder)

        lockname = 'lock-%s-%s-nightly' % (name, platform)
        lock = locks.SlaveLock(lockname)

        mozilla2_nightly_factory = factory.BuildFactory()
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['echo', WithProperties('Building on: %(slavename)s')],
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
          command=['echo', 'TinderboxPrint:', WithProperties('s: %(slavename)s')]
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['find', '.', '-maxdepth', '2', '-mtime', '+7', '-exec',
                     'rm', '-rfv', '{}', ';'],
            env=pf['env'],
            workdir='.',
            description=['cleanup', 'old', 'symbols'],
            flunkOnFailure=False,
        ))
        mozilla2_nightly_factory.addStep(Mercurial(
            branchType='inrepo',
            mode='clobber',
            repourl="%s%s" % (HGURL, hg_branch),
            timeout=60*60,
            retry=[60,3],
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, hg_branch)
        changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['python', 'client.py'] + branch['client_py_args'] + ['--cvsroot', cvsroot, 'checkout'],
            description=['running', 'client.py', 'checkout'],
            descriptionDone=['client.py', 'checkout'],
            haltOnFailure=True,
            timeout=60*60,
        ))
        mozilla2_nightly_factory.addStep(GetHgRevision(
            workdir='build/mozilla'
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, mozilla_branch)
        changesetLink += '/%(hg_revision)s title="Built from Mozilla revision %(hg_revision)s">moz:%(hg_revision)s</a>'
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['rm', '-rfv', 'configs'],
            description=['removing', 'configs'],
            descriptionDone=['remove', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['hg', 'clone', CONFIG_REPO_URL, 'configs'],
            description=['checking', 'out', 'configs'],
            descriptionDone=['checkout', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            # cp configs/thunderbird/$platform/mozconfig .mozconfig
            command=['cp', 'configs/%s/%s/%s' % (CONFIG_SUBDIR,
                                                 platform,
                                                 mozconfig
                                                 ),
                     '.mozconfig'],
            description=['copying', 'mozconfig'],
            descriptionDone=['copy', 'mozconfig'],
            haltOnFailure=True
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['cat', '.mozconfig'],
        ))

        buildcmd = 'build'
        mozilla2_nightly_factory.addStep(Compile(
            command=['make', '-f', 'client.mk', buildcmd],
            env=pf['env'],
            haltOnFailure=True,
            timeout=60*40
        ))
        if pf['upload_symbols']:
            mozilla2_nightly_factory.addStep(ShellCommand(
                command=['make', 'buildsymbols'],
                env=pf['env'],
                workdir='build/%s' % pf['platform_objdir'],
                haltOnFailure=True,
                timeout=60*30,
            ))
        if branch['package']:
                mozilla2_nightly_factory.addStep(ShellCommand(
                    command=['make', 'package'],
                    workdir='build/%s' % pf['platform_objdir'],
                    haltOnFailure=True
                ))
                if platform.startswith("win32"):
                    mozilla2_nightly_factory.addStep(ShellCommand(
                        command=['make', 'installer'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=True
                    ))
                mozilla2_nightly_factory.addStep(ShellCommand(
                        command=['make', 'package-compare'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=False
                    ))
                mozilla2_nightly_factory.addStep(ShellCommand(
                    command=['make', '-C',
                             '%s/mozilla/tools/update-packaging' % pf['platform_objdir']],
                    description=['create', 'complete', 'update'],
                    haltOnFailure=True
                ))
        mozilla2_nightly_factory.addStep(SetMozillaBuildProperties(
                objdir='build/%s/mozilla' % pf['platform_objdir']
        ))
        if upload_stage:
            mozilla2_nightly_factory.addStep(MozillaStageUpload(
                    objdir='%s/mozilla' % pf['platform_objdir'],
                    username=branch.get('stage_username',STAGE_USERNAME),
                    milestone=milestone,
                    remoteHost=branch.get('stage_server',STAGE_SERVER),
                    remoteBasePath=stage_base_path,
                    packageGlob=upload_glob,
                    platform=realPlatform,
                    group=branch.get('stage_group',STAGE_GROUP),
                    sshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    releaseToDated=True,
                    releaseToLatest=True,
                    releaseToTinderboxBuilds=True,
                    uploadCompleteMar=branch.get('upload_complete_mar', True),
                    tinderboxBuildsDir='%s-%s' % (milestone, platform),
                    dependToDated=True
            ))

        create_snippet = pf.get('create_snippet', branch['create_snippet'])
        aus = getConfig(DEFAULTS, branch, 'aus')

        if create_snippet:
	    # this is a tad ugly because we need to python interpolation
            # as well as WithProperties
            # here's an example of what it translates to:
            # /opt/aus2/build/0/Thunderbird/mozilla2/WINNT_x86-msvc/2008010103/en-US
            AUS2_FULL_UPLOAD_DIR = '%s/%s/%%(buildid)s/en-US' % \
              (branch['aus2_base_upload_dir'],
               branch['platforms'][platform]['update_platform'])
            mozilla2_nightly_factory.addStep(CreateCompleteUpdateSnippet(
                objdir='build/%s/mozilla' % pf['platform_objdir'],
                milestone=milestone,
                baseurl='%s/nightly' % download_base_url,
            ))
	    # New style aus configuration
	    if aus:
		uploadUpdateSnippet(mozilla2_nightly_factory, aus, pf)
	    
            mozilla2_nightly_factory.addStep(ShellCommand(
                command=['ssh', '-l', aus2_user, aus2_host,
                         WithProperties('mkdir -p %s' % AUS2_FULL_UPLOAD_DIR)],
                description=['create', 'aus2', 'upload', 'dir'],
                haltOnFailure=False,
                flunkOnFailure=False,
            ))
            mozilla2_nightly_factory.addStep(ShellCommand(
                command=['scp', '-o', 'User=%s' % aus2_user,
                         'dist/update/complete.update.snippet',
                         WithProperties('%s:%s/complete.txt' % \
                           (aus2_host, AUS2_FULL_UPLOAD_DIR))],
                workdir='build/%s/mozilla' % pf['platform_objdir'],
                description=['upload', 'complete', 'snippet'],
                haltOnFailure=False,
                flunkOnFailure=False,
            ))
        if pf['upload_symbols']:
            mozilla2_nightly_factory.addStep(ShellCommand(
                command=['make', 'uploadsymbols'],
                env=pf['env'],
                workdir='build/%s' % pf['platform_objdir'],
                haltOnFailure=True
            ))

        # in order to save disk space we clobber 'build' at the end.
        # we don't want to clobber anything else though, because symbols
        # sit alongside
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['rm', '-rfv', 'build'],
            env=pf['env'],
            workdir='.'
        ))

        mozilla2_nightly_builder = {
            'name': '%s nightly' % pf['base_name'],
            'slavenames': pf['slaves'],
            'builddir': '%s-%s-nightly' % (name, platform),
            'factory': mozilla2_nightly_factory,
            'category': name,
            'locks': [lock],
        }
        if nightly:
            c['builders'].append(mozilla2_nightly_builder)

        if l10n:
            mozilla2_l10n_repack_factory = CCNightlyRepackFactory(
                    hgHost='hg.mozilla.org',
                    tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                    project=branch.get('stage_project',appname),
                    appName=product,
                    enUSBinaryURL='%s/nightly/latest-%s/' % (download_base_url, milestone),
                    stageServer=branch.get('stage_server',STAGE_SERVER),
                    stageUsername=branch.get('stage_username',STAGE_USERNAME),
                    stageSshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    branchName=branch_name,
                    repoPath=hg_branch,
                    mozRepoPath=mozilla_branch,
                    l10nRepoPath=getConfig(DEFAULTS, branch, 'l10n_repo'),
                    buildToolsRepoPath='build/tools',
                    compareLocalesRepoPath='build/compare-locales',
                    compareLocalesTag='RELEASE_AUTOMATION',
                    buildSpace=None,
                    clobberURL=clobber_url,
                    clobberTime=30*24*7, #Hours before clobber
            )

            mozilla2_l10n_repack_builder = {
                'name': '%s %s l10n' % (product_name, pf['base_name']),
                'slavenames': pf['slaves'],
                'builddir': '%s-%s-l10n-full' % (name, platform),
                'factory': mozilla2_l10n_repack_factory,
                'category': '%s-l10n' % name,
            }
            c['builders'].append(mozilla2_l10n_repack_builder)
            c['schedulers'].append(NightlyL10n(
                                   name='%s %s l10n nightly scheduler' % (product_name, pf['base_name']),
                                   platform=realPlatform,
                                   tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                                   hour=[7],
                                   builderNames=[ '%s %s l10n' % (product_name, pf['base_name']) ],
                                   repoType='hg',
                                   branch=hg_branch,
                                   baseTag='default',
                                   localesFile='%s/locales/all-locales' % product,
                                  )) 

            mozilla2_l10n_repack_dep_factory = CCNightlyRepackFactory(
                    hgHost='hg.mozilla.org',
                    tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                    project=branch.get('stage_project',appname),
                    appName=product,
                    nightly=False, 
                    enUSBinaryURL='%s/nightly/latest-%s/' % (download_base_url, milestone), 
                    stageServer=branch.get('stage_server',STAGE_SERVER),
                    stageUsername=branch.get('stage_username',STAGE_USERNAME),
                    stageSshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    branchName=branch_name,
                    repoPath=hg_branch,
                    mozRepoPath=mozilla_branch,
                    l10nRepoPath=getConfig(DEFAULTS, branch, 'l10n_repo'),
                    buildToolsRepoPath='build/tools',
                    compareLocalesRepoPath='build/compare-locales',
                    compareLocalesTag='RELEASE_AUTOMATION',
                    buildSpace=None,
                    clobberURL=clobber_url,
                    clobberTime=30*24*7, #Hours before clobber
            )
            mozilla2_l10n_repack_dep_builder = {
                'name': '%s %s l10n build' % (product_name, pf['base_name']),
                'slavenames': pf['slaves'],
                'builddir': '%s-%s-l10n-dep' % (name, platform),
                'factory': mozilla2_l10n_repack_dep_factory,
                'category': '%s-l10n' % name,
            }
            c['builders'].append(mozilla2_l10n_repack_dep_builder)
         



####### STATUS TARGETS

from buildbot.status import html

baseport=8011

c['status'].append(html.WebStatus(
    http_port=baseport, allowForce=True
))

c['status'].append(html.WebStatus(
    http_port=8014, allowForce=False
))

####### PROJECT IDENTITY

# the 'projectName' string will be used to describe the project that this
# buildbot is working on. For example, it is used as the title of the
# waterfall HTML page. The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.

c['projectName'] = "Thunderbird:hg"
c['buildbotURL'] = "http://build.mozillamessaging.com/buildbot/production/"


#XXX
#c['schedulers'] = []

def mergeRequests(builder, req1, req2):
    import copy
    r1 = copy.copy(req1.source)
    r2 = copy.copy(req2.source)
    import sys
    print >> sys.stderr, "XXX: mergeRequests(%s,%s,%s)" % (builder, r1, r2)
    print >> sys.stderr, "XXX: r1.branch=%s r2.branch=%s" % (r1.branch, r2.branch)
 #   r1.branch = r2.branch 
 #   print >> sys.stderr, "XXX: making both branches be %s" % r1.branch
    rv = r1.canBeMergedWith(r2)
    print >> sys.stderr, "XXX: mergeRequests rv=%s" % rv
    #XXX: Busted, buildbot re-checks canBeMergedWith and asserts, so can't merge effectively
    #XXX: what buildbot thinks are non-mergeable requests, grr.
    return rv

##c['mergeRequests'] = mergeRequests

 # Give certain builders priority over other builders                                                                                                    
def prioritizeBuilders(botmaster, builders):
    def sortkey(builder):
        priority = 0

        #Release above everything else
        if builder.builder_status.category == 'release':
            priority += 4

        #Nightly above most
        if builder.name.find('nightly') > 1:
            priority += 2

        # l10n builds are low-priority
        if builder.name.find('l10n') > 1:
            priority -= 2
        
        #Smaller tweaks
            
        if debug:
            print >> sys.stderr, "prioritizeBuilder(%s) priority: %d" % (builder.name, priority)

        return -priority, builder.getOldestRequestTime()

    builders.sort(key=sortkey)

    return builders
c['prioritizeBuilders'] = prioritizeBuilders