calendar/master.cfg
author 'John Ford <jhford@mozilla.com>'
Thu, 21 Oct 2010 14:58:09 -0700
changeset 3164 c5f1d20795a5945dd1d1cecedc56993c97e98c61
parent 2443 de9365543110730cc5d6d3a6e5f624f26cca03da
child 3313 fadf1e2b4c471aa7563f890adada87d315def1d3
permissions -rw-r--r--
bug 602120 - new master for n900s r=aki p=jhford

# -*- python -*-
# ex: set syntax=python:

# Shorthand
c = BuildmasterConfig = {}

####### BUILDSLAVES

import BuildSlaves
reload(BuildSlaves)
c['slaves'] = BuildSlaves.SlaveList

# 'slavePortnum' defines the TCP port to listen on. This must match the value
# configured into the buildslaves (with their --master option)

c['slavePortnum'] = 9030

def getConfig(branch, value, default=None):
    return branch.get(value, DEFAULTS.get(value, default))

# most of the config is in an external file
import config
reload(config)
from config import *

import os
debug = os.environ.get('DEBUG')

if debug:
    import sys

c['builders'] = []
c['schedulers'] = []
c['change_source'] = []
c['status'] = []

####### Release Automation

import release_master
reload(release_master)

c['builders'].extend(release_master.builders)
c['schedulers'].extend(release_master.schedulers)
c['change_source'].extend(release_master.change_source)
c['status'].extend(release_master.status)

from buildbot import locks
from buildbot.process import factory
from buildbot.scheduler import Scheduler, Nightly, Periodic, Triggerable, AnyBranchScheduler
from buildbot.status.tinderbox import TinderboxMailNotifier
from buildbot.status.words import IRC
import buildbot.steps.source
from buildbot.steps.source import Mercurial
from buildbot.steps import trigger
from buildbot.steps.shell import Compile, ShellCommand, WithProperties, SetProperty, TreeSize

import buildbotcustom.changes.hgpoller
import buildbotcustom.steps.misc
import buildbotcustom.steps.test
import buildbotcustom.steps.transfer
import buildbotcustom.steps.updates
#import buildbotcustom.status.tinderbox

from buildbotcustom.steps.test import AliveTest, CompareBloatLogs, \
  CompareLeakLogs, Codesighs

import buildbotcustom.steps.misc
reload(buildbotcustom.steps.misc)
from buildbotcustom.steps.misc import SetMozillaBuildProperties, GetHgRevision, MozillaClobberer

from buildbotcustom.steps.transfer import MozillaStageUpload
from buildbotcustom.steps.updates import CreateCompleteUpdateSnippet

import buildbotcustom.process.factory
from buildbotcustom.process.factory import CCUnittestBuildFactory, CCNightlyRepackFactory

# l10n parallelization logic
import buildbotcustom.log
import buildbotcustom.l10n
from buildbotcustom.l10n import NightlyL10n, Scheduler as SchedulerL10n

import mozillaleak
reload(mozillaleak)
from mozillaleak import addLeakTestSteps

import mozillabuild
reload(mozillabuild)
from mozillabuild import *

# Keep track of our l10n pollers to eliminate dupes
hg_all_locales_poller_map = {}

####### L10n Scheduler with multiple dispatchers

# for debugging purposes of the repack on locale change system
buildbotcustom.log.init(
    scheduler = buildbotcustom.log.DEBUG,
    dispatcher = buildbotcustom.log.DEBUG
)

s = SchedulerL10n("l10n", "l10nbuilds.ini")
c['schedulers'].append(s)

####### SOURCES

setupHGPollersFromBranches(DEFAULTS, BRANCHES, c['change_source'])

for name in sorted(BRANCHES.keys()):
    # shorthand 
    branch = BRANCHES[name]
    build_factory = getConfig(DEFAULTS, branch, 'factory')
    mozilla_branch = getConfig(DEFAULTS, branch, 'mozilla_central_branch')
    l10n = getConfig(DEFAULTS, branch,'l10n')
    builders = []
    nightlyBuilders = []
    localeBuilders = []
    allBuilders = []
    
    if debug:
        print >> sys.stderr, "Branch: %s, Factory %s, L10n=%s" % (name, build_factory,l10n)

    nightly = branch.get('nightly') != False
    
    product = branch.get('product', PRODUCT)
    branch['product'] = product
    appname=branch.get('appname', MOZ_APP_NAME)
    branch['appname']= appname
    brandname=getConfig(DEFAULTS, branch, 'brand_name')
    product_name=getConfig(DEFAULTS, branch, 'product_name')
    branch['brand_name'] = brandname

    branch_name = getConfig(DEFAULTS, branch, 'branch_name')
    nightly_hour = getConfig(DEFAULTS, branch, 'nightly_hour')
    nightly_minute = getConfig(DEFAULTS, branch, 'nightly_minute')
    
    mozconfig=branch.get('mozconfig','mozconfig')
    aus2_host = branch.get('aus2_host', AUS2_HOST)
    aus2_user = branch.get('aus2_user', AUS2_USER)
    download_base_url = branch.get('download_base_url', DOWNLOAD_BASE_URL)
    stage_base_path = getConfig(DEFAULTS, branch, 'stage_base_path')
    builder_type = getConfig(DEFAULTS, branch, 'builder_type')
    
    # generate a list of builders, nightly builders (names must be different)
    # for easy access
    for platform in branch['platforms'].keys():
        platform_l10n = branch['platforms'][platform].get('l10n', l10n)
        if debug: 
            print >> sys.stderr, "Branch: %s Nightly: %s" % (name, nightly)
        
        builders.append('%s %s' % (branch['platforms'][platform]['base_name'], builder_type))
        
        # Not everybody wants nightlies
        if nightly:
            nightlyBuilders.append('%s nightly' % \
                                    branch['platforms'][platform]['base_name'])
        # Not everybody wants l10n repacks
        if platform_l10n:
            l10n_builder_name = "%s %s l10n" % (product_name, branch['platforms'][platform]['base_name'])
            localeBuilders.append("%s" % (l10n_builder_name))
            localeBuilders.append("%s build" % (l10n_builder_name))

    allBuilders.extend(builders)
    allBuilders.extend(nightlyBuilders)

    error_parser = "unix"
    if builder_type == "check" or builder_type == "bloat":
        error_parser = "unittest"

    # Currently, each branch goes to a different tree
    # If this changes in the future this may have to be
    # moved out of the loop
    tinderbox_tree = getConfig(DEFAULTS, branch, 'tinderbox_tree')
    c['status'].append(TinderboxMailNotifier(
        fromaddr="gozer@mozillamessaging.com",
        tree=tinderbox_tree,
        extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org", ],
        relayhost="mx.mozillamessaging.com",
        builders=allBuilders,
        logCompression="bzip2",
        errorparser = error_parser, 
    ))
    if l10n:
        # This notifies all l10n related build objects to Mozilla-l10n
        c['status'].append(TinderboxMailNotifier(
            fromaddr="gozer@mozillamessaging.com",
            tree="Mozilla-l10n",
            extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
            relayhost="mx.mozillamessaging.com",
            logCompression="bzip2",
            builders=localeBuilders,
            binaryURL="%s/nightly/latest-%s-l10n/" % (download_base_url, branch_name),
        ))
        # We only want the builds from the specified builders
        # since their builds have a build property called "locale"
        c['status'].append(TinderboxMailNotifier(
            fromaddr="gozer@mozillamessaging.com",
            tree=WithProperties("Mozilla-l10n-%(locale)s"),
            extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
            relayhost="mx.mozillamessaging.com",
            logCompression="bzip2",
            builders=localeBuilders,
            binaryURL="%s/nightly/latest-%s-l10n/" % (download_base_url, branch_name),
        ))
        

    if DEFAULTS['irc'] and branch.get('irc_nick') and branch.get('irc_channels'):
        c['status'].append(IRC(
            host='irc.mozilla.org',
            nick=branch['irc_nick'],
            channels=branch['irc_channels'],
            categories=[name],
        ))
    
    # Hg branch defaults to the branch name, or otherwise specified by hg_branch
    hg_branch = branch.get('hg_branch',name)

    # schedulers
    # this one gets triggered by the HG Poller
    add_poll_branches = getConfig(DEFAULTS, branch, 'add_poll_branches')
    import sys
    stable_timer = 5*60
#   if name.find('bloat') > 0:
#       stable_timer = 60*60*12 #XXX: Disable
#       print >> sys.stderr, "Scheduler is %s" % name

    c['schedulers'].append(AnyBranchScheduler(
        name=name,
        branches=[hg_branch, mozilla_branch] + add_poll_branches,
        treeStableTimer=stable_timer,
#        treeStableTimer=60*60*12, #XXX: Disable
        builderNames=builders
    ))
    if debug:
        print >> sys.stderr, "AnyBranchScheduler for %s watches %s" % (name, [hg_branch, mozilla_branch] + add_poll_branches)
        print >> sys.stderr, "hg_branch=%s for %s" % (hg_branch, name)
    print >> sys.stderr , "%s Nightly at %s:%s" % ( name, nightly_hour, nightly_minute)
    c['schedulers'].append(Nightly(
        name='%s nightly' % name,
        branch=hg_branch,
        hour=nightly_hour,
        minute=nightly_minute, 
        builderNames=nightlyBuilders
    ))
    if debug:
        print >> sys.stderr, "Periodic(name=%s buildernames=%s branch=%s BHuildTimer=%s)" %(name, builders, hg_branch, getConfig(DEFAULTS, branch, 'period'))
    c['schedulers'].append(Periodic(
        name='%s periodic' % name,
        builderNames=builders,
        branch=hg_branch,
        periodicBuildTimer=getConfig(DEFAULTS, branch, 'period'),
    ))
    if debug:
        print >> sys.stderr, "L10n=%s for Name: %s" % (l10n, name)
    if l10n:
        # We need to make sure we don't create duplicate l10n pollers
        poller_index = getConfig(DEFAULTS, branch, 'l10n_repo')
        poller_key = "%s %s" % (HGURL, poller_index)
        if not hg_all_locales_poller_map.get(poller_key):
            hg_all_locales_poller = HgAllLocalesPoller(hgURL = HGURL,
                            repositoryIndex =  poller_index,
                            pollInterval = 5*60)
            hg_all_locales_poller.parallelRequests = 1
            c['change_source'].append(hg_all_locales_poller)
            hg_all_locales_poller_map[poller_key]=1

    for platform in branch['platforms'].keys():
        # no need to have a whole new directory just for logs.
        # let's put these in their non-debug counterpart
        # translates to, eg:
        #  /home/ftp/pub/calendar/sunbird/tinderbox-builds/comm-central-linux

        #XXX: Kludge
        realPlatform = platform.replace('-debug', '').replace('64', '').replace('-shark', '')
        realPlatform = realPlatform.replace('-10.5','')

        logUploadDir = 'tinderbox-builds/%s-%s/' % (name, platform)

        lockname = 'lock-%s-%s-dep' % (name, platform)
        lock = locks.SlaveLock(lockname)

        # shorthand
        pf = branch['platforms'][platform]

        # Milestone controls where things get uploaded on stage
        milestone = pf.get('milestone',  branch.get('milestone', DEFAULTS.get('milestone', name)))
        if debug:
            print "Branch: %s Platform: %s Milestone: %s" % (name, platform, milestone)

        codesighs = pf.get('codesighs', getConfig(DEFAULTS, branch, 'codesighs'))
        upload_stage = pf.get('upload_stage', getConfig(DEFAULTS, branch, 'upload_stage'))
        l10n = pf.get('l10n', getConfig(DEFAULTS, branch, 'l10n'))
        build_space = pf.get('build_space', getConfig(DEFAULTS, branch, 'build_space'))
        cvsroot = pf.get('cvsroot', getConfig(DEFAULTS, branch, 'cvsroot'))
        
        upload_glob = None
        if branch.get('upload_glob'):
            upload_glob="%s/%s" % (pf['platform_objdir'], branch['upload_glob'])
        if pf.get('upload_glob'):
            upload_glob="%s/%s" % (pf['platform_objdir'], pf['upload_glob'])

        #Clobber support
        clobber_url = getConfig(DEFAULTS, branch, 'clobber_url')
        build_tools_repo = getConfig(DEFAULTS, branch, 'build_tools_repo')

        if build_factory == 'CCUnittestBuildFactory':
            client_py_args = getConfig(DEFAULTS, branch, 'client_py_args')
            client_py_extra_args = getConfig(DEFAULTS, branch, 'client_py_extra_args')
            mozmill = getConfig(DEFAULTS, branch, 'mozmill')
            if mozmill and platform in ('linux'):
		print >> sys.stderr, "Disabling enabled mozmill for %s" % platform
                mozmill = False

            if debug:
                print >> sys.stderr, "CCUnittestBuildFactory!!"
                print >> sys.stderr, "Builders are: %s" % builders
            unittest_factory = CCUnittestBuildFactory(
       		    branchName=branch_name,
                    platform=platform,
                    config_repo_path='build/buildbot-configs',
                    config_dir=CONFIG_SUBDIR,
                    objdir='objdir',
                    productName=appname,
                    brandName=brandname,
                    client_py_extra_args=client_py_extra_args,
                    mochitest_leak_threshold=None,
                    mochichrome_leak_threshold=None,
                    mochibrowser_leak_threshold=None,
                    hgHost=HGHOST,
                    repoPath=hg_branch,
                    mozRepoPath=mozilla_branch,
                    buildToolsRepoPath=build_tools_repo,
                    buildSpace=build_space,
                    clobberURL=clobber_url,
                    clobberTime=30*24*7, #Hours before clobber
                    buildsBeforeReboot=None,
                    exec_reftest_suites=False,
                    exec_mochi_suites=False,
                    exec_mozmill_suites=mozmill,
                )
            
            builder = {
                'name': '%s %s' % (pf['base_name'], builder_type),
                'slavenames': pf['slaves'],
                'builddir': "%s-%s-%s" % (platform, branch_name, builder_type),
                'factory': unittest_factory,
                'category': name,
            }
            c['builders'].append(builder)
            
            continue

        # The mozilla2_dep_factory currently covers: Depend, nightly and bloat
        # builders.
        mozilla2_dep_factory = factory.BuildFactory()
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', WithProperties('Building on: %(slavename)s')],
            env=pf['env']
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties('s: %(slavename)s')]
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command="rm -rfv %s/dist/%s-* %s/dist/install/sea/*.exe " %
                     (pf['platform_objdir'],
                      getConfig(DEFAULTS, branch, 'appname'),
                      pf['platform_objdir']),
                    
            env=pf['env'],
            description=['deleting', 'old', 'package'],
            descriptionDone=['delete', 'old', 'package']
        ))
        
        mozilla2_dep_factory.addStep(ShellCommand,
         command=['rm', '-rf', 'tools'],
         description=['clobber', 'build tools'],
         workdir='.'
        )
        mozilla2_dep_factory.addStep(ShellCommand,
         command=['bash', '-c',
          'if [ ! -d tools ]; then hg clone %s%s; fi' % (HGURL, build_tools_repo)],
         description=['clone', 'build tools'],
         workdir='.',
         flunkOnFailure=False,
         haltOnFailure=False,
        )

        # We need the basename of the current working dir so we can
        # ignore that dir when purging builds later.
        mozilla2_dep_factory.addStep(SetProperty,
         name='basename_pwd',
         command=['bash', '-c', 'basename "$PWD"'],
         property='builddir',
         workdir='.'
        )

        if build_space > 0:
            command = ['python', 'tools/buildfarm/maintenance/purge_builds.py',
                 '-s', str(build_space)]

            # Ignore the current dir also.
            command.extend(["-n", WithProperties("%(builddir)s")])
            command.append("..")

            mozilla2_dep_factory.addStep(ShellCommand,
             name='clean_old_builds',
             command=command,
             description=['cleaning', 'old', 'builds'],
             descriptionDone=['clean', 'old', 'builds'],
             haltOnFailure=True,
             workdir='.',
             timeout=3600, # One hour, because Windows is slow
            )

        clobberer_workdir = 'build/%s' % pf['platform_objdir']
        
        #XXX: This is a little brittle, assumes only objdir with ppc in them are 2-leveled     
        import os.path
        if platform.startswith("macosx") and clobberer_workdir.find('ppc') != -1:
            clobberer_workdir = os.path.dirname(clobberer_workdir)
        if platform.startswith("macosx") and clobberer_workdir.find('i386') != -1:
            clobberer_workdir =  os.path.dirname(clobberer_workdir)
        
        mozilla2_dep_factory.addStep(MozillaClobberer,
            branch=branch_name,
            clobber_url=clobber_url,
            clobberTime = 30*24*7, #Hours before clobber
            clobberer_path=WithProperties('%s/tools/clobberer/clobberer.py', 'builddir'),
            flunkOnFailure=False,
            haltOnFailure=False,
            timeout=3600, # One hour, because Windows is slow
        )
        #Clobber support end


        mozilla2_dep_factory.addStep(Mercurial(
            branchType='inrepo',
            mode='update',
            repourl="%s%s" % (HGURL, hg_branch),
            alwaysUseLatest=True,
            timeout=60*60,
            retry=[60,3],
            clobberOnBranchChange=False,
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, hg_branch)
        changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['python', 'client.py' ] + branch['client_py_args'] + ['--cvsroot', cvsroot, 'checkout'],
            description=['running', 'client.py', 'checkout'],
            descriptionDone=['python', 'client.py', 'checkout'],
            haltOnFailure=True,
            timeout=60*60,
        ))
        mozilla2_dep_factory.addStep(GetHgRevision(
            workdir='build/mozilla'
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, mozilla_branch)

        link_tag = 'moz'
        if mozilla_branch == 'mozilla-central':
            link_tag = 'm-c'
        changesetLink += '/%(hg_revision)s title="Built from Mozilla revision %(hg_revision)s">' + link_tag + ':%(hg_revision)s</a>'

        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['rm', '-rfv', 'configs'],
            description=['removing', 'configs'],
            descriptionDone=['remove', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['hg', 'clone', CONFIG_REPO_URL, 'configs'],
            description=['checking', 'out', 'configs'],
            descriptionDone=['checkout', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            # cp configs/$subdir/$platform/$branchName/mozconfig .mozconfig
            command=['cp', 'configs/%s/%s/%s/%s' % (CONFIG_SUBDIR,
                                                 platform,
                                                 branch_name,
                                                 mozconfig
                                                 ),
                     '.mozconfig'],
            description=['copying', 'mozconfig'],
            descriptionDone=['copy', 'mozconfig'],
            haltOnFailure=True
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['cat', '.mozconfig'],
        ))

        buildcmd = 'build'
        mozilla2_dep_factory.addStep(Compile(
            command=['make', '-f', 'client.mk', buildcmd],
            env=pf['env'],
            haltOnFailure=True,
            timeout=60*60
        ))

        # TODO change to calendar, when we support leak tests
        if branch.get('leak'):
            addLeakTestSteps(mozilla2_dep_factory, branch, pf, platform, 'thunderbird', logUploadDir, DEFAULTS)

        if platform.find('debug') == -1:
            if branch['package']:
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['make', 'package'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=True
                    ))
                    if platform.startswith("win32"):
                        mozilla2_dep_factory.addStep(ShellCommand(
                            command=['make', 'installer'],
                            workdir='build/%s' % pf['platform_objdir'],
                            haltOnFailure=True
                        ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['make', 'package-compare'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=False
                    ))
            mozilla2_dep_factory.addStep(SetMozillaBuildProperties(
                objdir='build/%s/mozilla' % pf['platform_objdir']
            ))
            if upload_stage:
                mozilla2_dep_factory.addStep(MozillaStageUpload(
                    objdir='%s/mozilla' % pf['platform_objdir'],
                    username=branch.get('stage_username',STAGE_USERNAME),
                    milestone=milestone,
                    remoteHost=branch.get('stage_server',STAGE_SERVER),
                    remoteBasePath=stage_base_path,
                    packageGlob=upload_glob,
                    platform=realPlatform,
                    group=branch.get('stage_group',STAGE_GROUP),
                    sshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    releaseToLatest=False,
                    releaseToDated=False,
                    releaseToTinderboxBuilds=True,
                    tinderboxBuildsDir='%s-%s' % (milestone, platform),
                    dependToDated=True
                ))
            if codesighs:
                if platform.find('win32') == -1 and platform.find('linux64') == -1:
                    # Codesighs
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['make'],
                        workdir='build/%s/mozilla/tools/codesighs' % pf['platform_objdir']
                    ))
                    # TODO change to calendar when we support codesighs
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['wget', '-O', 'codesize-auto-old.log',
                         'http://%s/pub/mozilla.org/thunderbird/%s/codesize-auto.log' %\
                          (branch.get('stage_server',STAGE_SERVER), logUploadDir)],
                        env=pf['env']
                    ))
                    mozilla2_dep_factory.addStep(Codesighs(
                        objdir='../%s/mozilla' % pf['platform_objdir'],
                        platform=realPlatform,
                        workdir='build/mozilla',
                        env=pf['env']
                    ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['cat', 'codesize-auto-diff.log']
                    ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['scp', '-o', 'User=%s' % branch.get('stage_username',STAGE_USERNAME),
                         '-o', 'IdentityFile=~/.ssh/%s' % branch.get('stage_ssh_key',STAGE_SSH_KEY),
                         'codesize-auto.log',
                         '%s:%s/%s' % (branch.get('stage_server',STAGE_SERVER), stage_base_path, logUploadDir)]
                    ))

        mozilla2_dep_builder = {
            'name': '%s %s' % (pf['base_name'], builder_type),
            'slavenames': pf['slaves'],
            'builddir': '%s-%s' % (name, platform),
            'factory': mozilla2_dep_factory,
            'category': name,
            'locks' : [lock],
        }
        c['builders'].append(mozilla2_dep_builder)

        lockname = 'lock-%s-%s-nightly' % (name, platform)
        lock = locks.SlaveLock(lockname)

        mozilla2_nightly_factory = factory.BuildFactory()
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['echo', WithProperties('Building on: %(slavename)s')],
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
          command=['echo', 'TinderboxPrint:', WithProperties('s: %(slavename)s')]
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['find', '.', '-maxdepth', '2', '-mtime', '+7', '-exec',
                     'rm', '-rfv', '{}', ';'],
            env=pf['env'],
            workdir='.',
            description=['cleanup', 'old', 'symbols'],
            flunkOnFailure=False,
        ))

        mozilla2_nightly_factory.addStep(SetProperty,
         name='basename_pwd',
         command=['bash', '-c', 'basename "$PWD"'],
         property='builddir',
         workdir='.'
        )

        mozilla2_nightly_factory.addStep(ShellCommand,
         command=['rm', '-rf', 'tools'],
         description=['clobber', 'build tools'],
         workdir='.'
        )
        mozilla2_nightly_factory.addStep(ShellCommand,
         command=['bash', '-c',
          'if [ ! -d tools ]; then hg clone %s%s; fi' % (HGURL, build_tools_repo)],
         description=['clone', 'build tools'],
         workdir='.',
         flunkOnFailure=False,
         haltOnFailure=False,
        )

        if build_space > 0:
            command = ['python', 'tools/buildfarm/maintenance/purge_builds.py',
                 '-s', str(build_space)]

            # Ignore the current dir also.
            command.extend(["-n", WithProperties("%(builddir)s")])
            command.append("..")

            mozilla2_nightly_factory.addStep(ShellCommand,
             name='clean_old_builds',
             command=command,
             description=['cleaning', 'old', 'builds'],
             descriptionDone=['clean', 'old', 'builds'],
             haltOnFailure=True,
             workdir='.',
             timeout=3600, # One hour, because Windows is slow
            )

        mozilla2_nightly_factory.addStep(Mercurial(
            branchType='inrepo',
            mode='clobber',
            repourl="%s%s" % (HGURL, hg_branch),
            timeout=60*60,
            retry=[60,3],
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, hg_branch)
        changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        client_py_args = branch['client_py_args']
        if branch.get('nightly_client_py_args'):
            client_py_args = client_py_args + branch['nightly_client_py_args']
            import sys
            print >> sys.stderr, "client_py_args will be : %s" % client_py_args

        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['python', 'client.py'] + client_py_args + ['--cvsroot', cvsroot, 'checkout'],
            description=['running', 'client.py', 'checkout'],
            descriptionDone=['client.py', 'checkout'],
            haltOnFailure=True,
            timeout=60*60,
        ))
        mozilla2_nightly_factory.addStep(GetHgRevision(
            workdir='build/mozilla'
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, mozilla_branch)
        changesetLink += '/%(hg_revision)s title="Built from Mozilla revision %(hg_revision)s">moz:%(hg_revision)s</a>'
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['rm', '-rfv', 'configs'],
            description=['removing', 'configs'],
            descriptionDone=['remove', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['hg', 'clone', CONFIG_REPO_URL, 'configs'],
            description=['checking', 'out', 'configs'],
            descriptionDone=['checkout', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            # cp configs/$subdir/$platform/mozconfig .mozconfig
            command=['cp', 'configs/%s/%s/%s/%s' % (CONFIG_SUBDIR,
                                                 platform,
                                                 branch_name,
                                                 mozconfig
                                                 ),
                     '.mozconfig'],
            description=['copying', 'mozconfig'],
            descriptionDone=['copy', 'mozconfig'],
            haltOnFailure=True
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['cat', '.mozconfig'],
        ))

        buildcmd = 'build'
        mozilla2_nightly_factory.addStep(Compile(
            command=['make', '-f', 'client.mk', buildcmd],
            env=pf['env'],
            haltOnFailure=True,
            timeout=60*40
        ))
        if pf['upload_symbols']:
            mozilla2_nightly_factory.addStep(ShellCommand(
                command=['make', 'buildsymbols'],
                env=pf['env'],
                workdir='build/%s' % pf['platform_objdir'],
                haltOnFailure=True,
                timeout=60*60,
            ))
        if branch['package']:
                mozilla2_nightly_factory.addStep(ShellCommand(
                    command=['make', 'package'],
                    workdir='build/%s' % pf['platform_objdir'],
                    haltOnFailure=True
                ))
                if platform.startswith("win32"):
                    mozilla2_nightly_factory.addStep(ShellCommand(
                        command=['make', 'installer'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=True
                    ))
                mozilla2_nightly_factory.addStep(ShellCommand(
                        command=['make', 'package-compare'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=False
                    ))
                mozilla2_nightly_factory.addStep(ShellCommand(
                    command=['make', '-C',
                             '%s/mozilla/tools/update-packaging' % pf['platform_objdir']],
                    description=['create', 'complete', 'update'],
                    haltOnFailure=True
                ))
        mozilla2_nightly_factory.addStep(SetMozillaBuildProperties(
                objdir='build/%s/mozilla' % pf['platform_objdir']
        ))
        if upload_stage:
            mozilla2_nightly_factory.addStep(MozillaStageUpload(
                    objdir='%s/mozilla' % pf['platform_objdir'],
                    username=branch.get('stage_username',STAGE_USERNAME),
                    milestone=milestone,
                    remoteHost=branch.get('stage_server',STAGE_SERVER),
                    remoteBasePath=stage_base_path,
                    packageGlob=upload_glob,
                    platform=realPlatform,
                    group=branch.get('stage_group',STAGE_GROUP),
                    sshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    releaseToDated=True,
                    releaseToLatest=True,
                    releaseToTinderboxBuilds=True,
                    uploadCompleteMar=branch.get('upload_complete_mar', True),
                    tinderboxBuildsDir='%s-%s' % (milestone, platform),
                    dependToDated=True
            ))

        create_snippet = pf.get('create_snippet', branch['create_snippet'])
        aus = getConfig(DEFAULTS, branch, 'aus')

        if create_snippet:
            mozilla2_nightly_factory.addStep(CreateCompleteUpdateSnippet(
                objdir='build/%s/mozilla' % pf['platform_objdir'],
                milestone=milestone,
                baseurl='%s/nightly' % download_base_url,
            ))
	    # New style aus configuration
            if aus:
                uploadUpdateSnippet(mozilla2_nightly_factory, aus, pf)
    
        if pf['upload_symbols']:
            mozilla2_nightly_factory.addStep(ShellCommand(
                command=['make', 'uploadsymbols'],
                env=pf['env'],
                workdir='build/%s' % pf['platform_objdir'],
                haltOnFailure=True
            ))
        if l10n:
           l10n_scheduler="%s %s %s l10n nightly scheduler" % (product_name, platform, branch_name)
           mozilla2_nightly_factory.addStep(trigger.Trigger(
               schedulerNames=[l10n_scheduler],
               waitForFinish=False,
               flunkOnFailure=False,
               haltOnFailure=False,
            ))

        # in order to save disk space we clobber 'build' at the end.
        # we don't want to clobber anything else though, because symbols
        # sit alongside
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['rm', '-rfv', 'build'],
            env=pf['env'],
            workdir='.'
        ))
        nightly_builder = pf.get('nightly_builder',  branch.get('nightly_builder', DEFAULTS.get('nightly_builder')))
        import sys
        nightly_slaves = pf['slaves']
        if nightly_builder:
            print >> sys.stderr, "%s nightly builder override: %s" % (pf['base_name'], nightly_builder)
            nightly_slaves = nightly_builder

        mozilla2_nightly_builder = {
            'name': '%s nightly' % pf['base_name'],
            'slavenames': nightly_slaves,
            'builddir': '%s-%s-nightly' % (name, platform),
            'factory': mozilla2_nightly_factory,
            'category': name,
            'locks': [lock],
        }
        if nightly:
            c['builders'].append(mozilla2_nightly_builder)

        if l10n:
            l10n_nightly_updates = pf.get('l10n_nightly_updates',  branch.get('l10n_nightly_updates', DEFAULTS.get('l10n_nightly_updates', name)))
            l10n_dated_dirs = False
            if l10n_nightly_updates:
                l10n_dated_dirs = True
            print >> sys.stderr, "branchName: %s, repoPath: %s" % (branch_name, hg_branch)
            mozilla2_l10n_repack_factory = CCNightlyRepackFactory(
                    nightly=True,
                    hgHost='hg.mozilla.org',
                    tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                    project=branch.get('stage_project',appname),
                    appName=product,
                    enUSBinaryURL='%s/nightly/latest-%s/' % (download_base_url, milestone),
                    downloadBaseURL=download_base_url,
                    stageServer=branch.get('stage_server',STAGE_SERVER),
                    stageUsername=branch.get('stage_username',STAGE_USERNAME),
                    stageSshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    ausBaseUploadDir=aus['base_upload_dir'],
                    ausUser=aus['user'],
                    ausHost=aus['host'],
                    updatePlatform=pf['update_platform'],
                    branchName=branch_name,
                    repoPath=hg_branch,
                    mozRepoPath=mozilla_branch,
                    l10nRepoPath=getConfig(DEFAULTS, branch, 'l10n_repo'),
                    l10nNightlyUpdate=l10n_nightly_updates,
                    l10nDatedDirs=l10n_dated_dirs,
                    buildToolsRepoPath='build/tools',
                    compareLocalesRepoPath='build/compare-locales',
                    compareLocalesTag='RELEASE_AUTOMATION',
                    buildSpace=build_space,
                    clobberURL=clobber_url,
                    clobberTime=30*24*7, #Hours before clobber
            )

            mozilla2_l10n_repack_builder = {
                'name': '%s %s l10n' % (product_name, pf['base_name']),
                'slavenames': pf['slaves'],
                'builddir': '%s-%s-l10n-full' % (name, platform),
                'factory': mozilla2_l10n_repack_factory,
                'category': '%s-l10n' % name,
            }
            c['builders'].append(mozilla2_l10n_repack_builder)
            c['schedulers'].append(NightlyL10n(
                                   name='%s %s %s l10n nightly scheduler' % (product_name, platform, branch_name),
                                   platform=realPlatform,
                                   tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                                   hour=[7],
                                   builderNames=[ '%s %s l10n' % (product_name, pf['base_name']) ],
                                   repoType='hg',
                                   branch=hg_branch,
                                   baseTag='default',
                                   localesFile='%s/locales/all-locales' % product,
                                  )) 

            mozilla2_l10n_repack_dep_factory = CCNightlyRepackFactory(
                    hgHost='hg.mozilla.org',
                    tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                    project=branch.get('stage_project',appname),
                    appName=product,
                    nightly=False, 
                    enUSBinaryURL='%s/nightly/latest-%s/' % (download_base_url, milestone), 
                    stageServer=branch.get('stage_server',STAGE_SERVER),
                    stageUsername=branch.get('stage_username',STAGE_USERNAME),
                    stageSshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    branchName=branch_name,
                    repoPath=hg_branch,
                    mozRepoPath=mozilla_branch,
                    l10nRepoPath=getConfig(DEFAULTS, branch, 'l10n_repo'),
                    buildToolsRepoPath='build/tools',
                    compareLocalesRepoPath='build/compare-locales',
                    compareLocalesTag='RELEASE_AUTOMATION',
                    buildSpace=build_space,
                    clobberURL=clobber_url,
                    clobberTime=30*24*7, #Hours before clobber
            )
            mozilla2_l10n_repack_dep_builder = {
                'name': '%s %s l10n build' % (product_name, pf['base_name']),
                'slavenames': pf['slaves'],
                'builddir': '%s-%s-l10n-dep' % (name, platform),
                'factory': mozilla2_l10n_repack_dep_factory,
                'category': '%s-l10n' % name,
            }
            c['builders'].append(mozilla2_l10n_repack_dep_builder)
         



####### STATUS TARGETS

from buildbot.status import html

baseport=8031

c['status'].append(html.WebStatus(
    http_port=baseport, allowForce=True
))

c['status'].append(html.WebStatus(
    http_port=8032, allowForce=False
))

####### PROJECT IDENTITY

# the 'projectName' string will be used to describe the project that this
# buildbot is working on. For example, it is used as the title of the
# waterfall HTML page. The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.

c['projectName'] = "Calendar"
c['buildbotURL'] = "http://build.mozillamessaging.com/buildbot/calendar"


#XXX
#c['schedulers'] = []

def mergeRequests(builder, req1, req2):
    import copy
    r1 = copy.copy(req1.source)
    r2 = copy.copy(req2.source)
    import sys
    print >> sys.stderr, "XXX: mergeRequests(%s,%s,%s)" % (builder, r1, r2)
    print >> sys.stderr, "XXX: r1.branch=%s r2.branch=%s" % (r1.branch, r2.branch)
 #   r1.branch = r2.branch 
 #   print >> sys.stderr, "XXX: making both branches be %s" % r1.branch
    rv = r1.canBeMergedWith(r2)
    print >> sys.stderr, "XXX: mergeRequests rv=%s" % rv
    #XXX: Busted, buildbot re-checks canBeMergedWith and asserts, so can't merge effectively
    #XXX: what buildbot thinks are non-mergeable requests, grr.
    return rv

##c['mergeRequests'] = mergeRequests

 # Give certain builders priority over other builders                                                                                                    
def prioritizeBuilders(botmaster, builders):
    def sortkey(builder):
        priority = 0

        #Release above everything else
        if builder.builder_status.category == 'release':
            priority += 4

        #Nightly above most
        if builder.name.find('nightly') > 1:
            priority += 2

        return -priority, builder.getOldestRequestTime()

    builders.sort(key=sortkey)

    return builders
c['prioritizeBuilders'] = prioritizeBuilders