calendar/master.cfg
author John Hopkins <jhopkins@mozilla.com>
Mon, 30 Apr 2012 09:49:11 -0400
changeset 5892 d344e17405c7b83018460f420c7da031f0d632c5
parent 5158 09652afdaf8e7f38885f5b5808471ff488320226
child 6321 dfe05d4f47f4ad67bde719985275201b77489d5b
permissions -rw-r--r--
copy Thunderbird l10n files under mozilla/ directory (Bug 711534 - Configure Thunderbird release builders) r=standard8

# -*- python -*-
# ex: set syntax=python:

# Shorthand
c = BuildmasterConfig = {}

####### BUILDSLAVES

import BuildSlaves
reload(BuildSlaves)
c['slaves'] = BuildSlaves.SlaveList

# 'slavePortnum' defines the TCP port to listen on. This must match the value
# configured into the buildslaves (with their --master option)

c['slavePortnum'] = 9030

def getConfig(branch, value, default=None):
    return branch.get(value, DEFAULTS.get(value, default))

def getExtensionConfig(ebranch, branch, value, default=None):
    return ebranch.get(value, branch.get(value, DEFAULTS.get(value, default)))

# most of the config is in an external file
import config
reload(config)
from config import *

import os
debug = os.environ.get('DEBUG')

if debug:
    import sys

c['builders'] = []
c['schedulers'] = []
c['change_source'] = []
c['status'] = []

####### Release Automation

import release_master
reload(release_master)

c['builders'].extend(release_master.builders)
c['schedulers'].extend(release_master.schedulers)
c['change_source'].extend(release_master.change_source)
c['status'].extend(release_master.status)

from buildbot import locks
from buildbot.process import factory
from buildbot.scheduler import Scheduler, Nightly, Periodic, Triggerable, AnyBranchScheduler
from buildbot.status.tinderbox import TinderboxMailNotifier
from buildbot.status.words import IRC
import buildbot.steps.source
from buildbot.steps.source import Mercurial
from buildbot.steps import trigger
from buildbot.steps.shell import Compile, ShellCommand, WithProperties, SetProperty, TreeSize

import buildbotcustom.changes.hgpoller
import buildbotcustom.steps.misc
import buildbotcustom.steps.test
import buildbotcustom.steps.transfer
import buildbotcustom.steps.updates
#import buildbotcustom.status.tinderbox

from buildbotcustom.steps.test import AliveTest, CompareBloatLogs, \
  CompareLeakLogs, Codesighs

import buildbotcustom.steps.misc
reload(buildbotcustom.steps.misc)
from buildbotcustom.steps.misc import SetMozillaBuildProperties, GetHgRevision, MozillaClobberer
import buildbotcustom.steps.unittest as unittest_steps
from buildbotcustom.steps.transfer import MozillaStageUpload
from buildbotcustom.steps.updates import CreateCompleteUpdateSnippet

import buildbotcustom.process.factory
from buildbotcustom.process.factory import CCUnittestBuildFactory, CCNightlyRepackFactory

import buildbotcustom.process.extensionfactory
reload(buildbotcustom.process.extensionfactory)
from buildbotcustom.process.extensionfactory import CCNightlyExtensionRepackFactory
# l10n parallelization logic
import buildbotcustom.log
import buildbotcustom.l10n
from buildbotcustom.l10n import NightlyL10n, Scheduler as SchedulerL10n

import mozillaleak
reload(mozillaleak)
from mozillaleak import addLeakTestSteps

import mozillabuild
reload(mozillabuild)
from mozillabuild import *

# Keep track of our l10n pollers to eliminate dupes
hg_all_locales_poller_map = {}

####### L10n Scheduler with multiple dispatchers

# for debugging purposes of the repack on locale change system
buildbotcustom.log.init(
    scheduler = buildbotcustom.log.DEBUG,
    dispatcher = buildbotcustom.log.DEBUG
)

s = SchedulerL10n("l10n", "l10nbuilds.ini")
c['schedulers'].append(s)

####### SOURCES

setupHGPollersFromBranches(DEFAULTS, BRANCHES, c['change_source'])

for name in sorted(BRANCHES.keys()):
    # shorthand 
    branch = BRANCHES[name]
    build_factory = getConfig(DEFAULTS, branch, 'factory')
    mozilla_branch = getConfig(DEFAULTS, branch, 'mozilla_central_branch')
    l10n = getConfig(DEFAULTS, branch,'l10n')
    builders = []
    nightlyBuilders = []
    localeBuilders = []
    allBuilders = []
    
    if debug:
        print >> sys.stderr, "Branch: %s, Factory %s, L10n=%s" % (name, build_factory,l10n)

    nightly = branch.get('nightly') != False
    
    product = branch.get('product', PRODUCT)
    branch['product'] = product
    appname=branch.get('appname', MOZ_APP_NAME)
    branch['appname']= appname
    brandname=getConfig(DEFAULTS, branch, 'brand_name')
    product_name=getConfig(DEFAULTS, branch, 'product_name')
    branch['brand_name'] = brandname

    branch_name = getConfig(DEFAULTS, branch, 'branch_name')
    nightly_hour = getConfig(DEFAULTS, branch, 'nightly_hour')
    nightly_minute = getConfig(DEFAULTS, branch, 'nightly_minute')
    
    mozconfig=branch.get('mozconfig','mozconfig')
    aus2_host = branch.get('aus2_host', AUS2_HOST)
    aus2_user = branch.get('aus2_user', AUS2_USER)
    download_base_url = branch.get('download_base_url', DOWNLOAD_BASE_URL)
    stage_base_path = getConfig(DEFAULTS, branch, 'stage_base_path')
    builder_type = getConfig(DEFAULTS, branch, 'builder_type')
    
    # generate a list of builders, nightly builders (names must be different)
    # for easy access
    for platform in branch['platforms'].keys():
        platform_l10n = branch['platforms'][platform].get('l10n', l10n)
        if debug: 
            print >> sys.stderr, "Branch: %s Nightly: %s" % (name, nightly)
        
        builders.append('%s %s' % (branch['platforms'][platform]['base_name'], builder_type))
        
        # Not everybody wants nightlies
        if nightly:
            nightlyBuilders.append('%s nightly' % \
                                    branch['platforms'][platform]['base_name'])
        # Not everybody wants l10n repacks
        if platform_l10n:
            l10n_builder_name = "%s %s l10n" % (product_name, branch['platforms'][platform]['base_name'])
            localeBuilders.append("%s" % (l10n_builder_name))
            localeBuilders.append("%s build" % (l10n_builder_name))

    allBuilders.extend(builders)
    allBuilders.extend(nightlyBuilders)

    error_parser = "unix"
    if builder_type == "check" or builder_type == "bloat":
        error_parser = "unittest"

    # Currently, each branch goes to a different tree
    # If this changes in the future this may have to be
    # moved out of the loop
    tinderbox_tree = getConfig(DEFAULTS, branch, 'tinderbox_tree')
    c['status'].append(TinderboxMailNotifier(
        fromaddr="gozer@mozillamessaging.com",
        tree=tinderbox_tree,
        extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org", ],
        relayhost="mx.mozillamessaging.com",
        builders=allBuilders,
        logCompression="bzip2",
        errorparser = error_parser, 
    ))
    if l10n:
        # This notifies all l10n related build objects to Mozilla-l10n
        c['status'].append(TinderboxMailNotifier(
            fromaddr="gozer@mozillamessaging.com",
            tree="Mozilla-l10n",
            extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
            relayhost="mx.mozillamessaging.com",
            logCompression="bzip2",
            builders=localeBuilders,
            binaryURL="%s/nightly/latest-%s-l10n/" % (download_base_url, branch_name),
        ))
        # We only want the builds from the specified builders
        # since their builds have a build property called "locale"
        c['status'].append(TinderboxMailNotifier(
            fromaddr="gozer@mozillamessaging.com",
            tree=WithProperties("Mozilla-l10n-%(locale)s"),
            extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
            relayhost="mx.mozillamessaging.com",
            logCompression="bzip2",
            builders=localeBuilders,
            binaryURL="%s/nightly/latest-%s-l10n/" % (download_base_url, branch_name),
        ))
        

    if DEFAULTS['irc'] and branch.get('irc_nick') and branch.get('irc_channels'):
        c['status'].append(IRC(
            host='irc.mozilla.org',
            nick=branch['irc_nick'],
            channels=branch['irc_channels'],
            categories=[name],
        ))
    
    # Hg branch defaults to the branch name, or otherwise specified by hg_branch
    hg_branch = branch.get('hg_branch',name)

    # schedulers
    # this one gets triggered by the HG Poller
    add_poll_branches = getConfig(DEFAULTS, branch, 'add_poll_branches')
    import sys
    stable_timer = 5*60
#   if name.find('bloat') > 0:
#       stable_timer = 60*60*12 #XXX: Disable
#       print >> sys.stderr, "Scheduler is %s" % name

    c['schedulers'].append(AnyBranchScheduler(
        name=name,
        branches=[hg_branch, mozilla_branch] + add_poll_branches,
        treeStableTimer=stable_timer,
#        treeStableTimer=60*60*12, #XXX: Disable
        builderNames=builders
    ))
    if debug:
        print >> sys.stderr, "AnyBranchScheduler for %s watches %s" % (name, [hg_branch, mozilla_branch] + add_poll_branches)
        print >> sys.stderr, "hg_branch=%s for %s" % (hg_branch, name)
    print >> sys.stderr , "%s Nightly at %s:%s" % ( name, nightly_hour, nightly_minute)
    c['schedulers'].append(Nightly(
        name='%s nightly' % name,
        branch=hg_branch,
        hour=nightly_hour,
        minute=nightly_minute, 
        builderNames=nightlyBuilders
    ))
    if debug:
        print >> sys.stderr, "Periodic(name=%s buildernames=%s branch=%s BHuildTimer=%s)" %(name, builders, hg_branch, getConfig(DEFAULTS, branch, 'period'))
# No periodic schedulers necessary
#    if hg_branch != 'comm-central' and hg_branch != 'releases/comm-beta':
#        c['schedulers'].append(Periodic(
#            name='%s periodic' % name,
#            builderNames=builders,
#            branch=hg_branch,
#            periodicBuildTimer=getConfig(DEFAULTS, branch, 'period'),
#        ))
    if debug:
        print >> sys.stderr, "L10n=%s for Name: %s" % (l10n, name)
    if l10n:
        # We need to make sure we don't create duplicate l10n pollers
        poller_index = getConfig(DEFAULTS, branch, 'l10n_repo')
        poller_key = "%s %s" % (HGURL, poller_index)
        if not hg_all_locales_poller_map.get(poller_key):
            hg_all_locales_poller = HgAllLocalesPoller(hgURL = HGURL,
                            repositoryIndex =  poller_index,
                            pollInterval = 5*60)
            hg_all_locales_poller.parallelRequests = 1
            c['change_source'].append(hg_all_locales_poller)
            hg_all_locales_poller_map[poller_key]=1

    for platform in branch['platforms'].keys():
        # no need to have a whole new directory just for logs.
        # let's put these in their non-debug counterpart
        # translates to, eg:
        #  /home/ftp/pub/calendar/sunbird/tinderbox-builds/comm-central-linux

        #XXX: Kludge
        realPlatform = platform.replace('-debug', '').replace('64', '').replace('-shark', '')
        realPlatform = realPlatform.replace('-10.5','')
        realPlatform = realPlatform.replace('-10.6','')

        logUploadDir = 'tinderbox-builds/%s-%s/' % (name, platform)

        lockname = 'lock-%s-%s-dep' % (name, platform)
        lock = locks.SlaveLock(lockname)

        # shorthand
        pf = branch['platforms'][platform]

        # Milestone controls where things get uploaded on stage
        milestone = pf.get('milestone',  branch.get('milestone', DEFAULTS.get('milestone', name)))
        if debug:
            print "Branch: %s Platform: %s Milestone: %s" % (name, platform, milestone)

        codesighs = pf.get('codesighs', getConfig(DEFAULTS, branch, 'codesighs'))
        upload_stage = pf.get('upload_stage', getConfig(DEFAULTS, branch, 'upload_stage'))
        l10n = pf.get('l10n', getConfig(DEFAULTS, branch, 'l10n'))
        build_space = pf.get('build_space', getConfig(DEFAULTS, branch, 'build_space'))

        upload_glob = None
        if branch.get('upload_glob'):
            upload_glob="%s/%s" % (pf['platform_objdir'], branch['upload_glob'])
        if pf.get('upload_glob'):
            upload_glob="%s/%s" % (pf['platform_objdir'], pf['upload_glob'])

        #Clobber support
        clobber_url = getConfig(DEFAULTS, branch, 'clobber_url')
        build_tools_repo = getConfig(DEFAULTS, branch, 'build_tools_repo')

        if build_factory == 'CCUnittestBuildFactory':
            client_py_args = getConfig(DEFAULTS, branch, 'client_py_args')
            client_py_extra_args = getConfig(DEFAULTS, branch, 'client_py_extra_args')
            mozmill = getConfig(DEFAULTS, branch, 'mozmill')
            if mozmill and platform in ('linux'):
		print >> sys.stderr, "Disabling enabled mozmill for %s" % platform
                mozmill = False

            if debug:
                print >> sys.stderr, "CCUnittestBuildFactory!!"
                print >> sys.stderr, "Builders are: %s" % builders
            unittest_factory = CCUnittestBuildFactory(
       		    branchName=branch_name,
                    platform=platform,
                    config_repo_path='build/buildbot-configs',
                    config_dir=CONFIG_SUBDIR,
                    objdir='objdir',
                    productName=appname,
                    brandName=brandname,
                    client_py_extra_args=client_py_extra_args,
                    mochitest_leak_threshold=None,
                    mochichrome_leak_threshold=None,
                    mochibrowser_leak_threshold=None,
                    hgHost=HGHOST,
                    repoPath=hg_branch,
                    mozRepoPath=mozilla_branch,
                    buildToolsRepoPath=build_tools_repo,
                    buildSpace=build_space,
                    clobberURL=clobber_url,
                    clobberTime=30*24*7, #Hours before clobber
                    buildsBeforeReboot=None,
                    exec_reftest_suites=False,
                    exec_mochi_suites=False,
                    exec_mozmill_suites=mozmill,
                )
            
            builder = {
                'name': '%s %s' % (pf['base_name'], builder_type),
                'slavenames': pf['slaves'],
                'builddir': "%s-%s-%s" % (platform, branch_name, builder_type),
                'factory': unittest_factory,
                'category': name,
            }
            c['builders'].append(builder)
            
            continue

        # The mozilla2_dep_factory currently covers: Depend, nightly and bloat
        # builders.
        mozilla2_dep_factory = factory.BuildFactory()
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', WithProperties('Building on: %(slavename)s')],
            env=pf['env']
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties('s: %(slavename)s')]
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command="rm -rfv %s/dist/%s-* %s/dist/install/sea/*.exe " %
                     (pf['platform_objdir'],
                      getConfig(DEFAULTS, branch, 'appname'),
                      pf['platform_objdir']),
                    
            env=pf['env'],
            description=['deleting', 'old', 'package'],
            descriptionDone=['delete', 'old', 'package']
        ))
        
        mozilla2_dep_factory.addStep(ShellCommand,
         command=['rm', '-rf', 'tools'],
         description=['clobber', 'build tools'],
         workdir='.'
        )
        mozilla2_dep_factory.addStep(ShellCommand,
         command=['bash', '-c',
          'if [ ! -d tools ]; then hg clone %s%s; fi' % (HGURL, build_tools_repo)],
         description=['clone', 'build tools'],
         workdir='.',
         flunkOnFailure=False,
         haltOnFailure=False,
        )

        # We need the basename of the current working dir so we can
        # ignore that dir when purging builds later.
        mozilla2_dep_factory.addStep(SetProperty,
         name='basename_pwd',
         command=['bash', '-c', 'basename "$PWD"'],
         property='builddir',
         workdir='.'
        )

        if build_space > 0:
            command = ['python', 'tools/buildfarm/maintenance/purge_builds.py',
                 '-s', str(build_space)]

            # Ignore the current dir also.
            command.extend(["-n", WithProperties("%(builddir)s")])
            command.append("..")

            mozilla2_dep_factory.addStep(ShellCommand,
             name='clean_old_builds',
             command=command,
             description=['cleaning', 'old', 'builds'],
             descriptionDone=['clean', 'old', 'builds'],
             haltOnFailure=True,
             workdir='.',
             timeout=3600, # One hour, because Windows is slow
            )

        clobberer_workdir = 'build/%s' % pf['platform_objdir']
        
        #XXX: This is a little brittle, assumes only objdir with ppc in them are 2-leveled     
        import os.path
        if platform.startswith("macosx") and clobberer_workdir.find('ppc') != -1:
            clobberer_workdir = os.path.dirname(clobberer_workdir)
        if platform.startswith("macosx") and clobberer_workdir.find('i386') != -1:
            clobberer_workdir =  os.path.dirname(clobberer_workdir)
        
        mozilla2_dep_factory.addStep(MozillaClobberer,
            branch=branch_name,
            clobber_url=clobber_url,
            clobberTime = 30*24*7, #Hours before clobber
            clobberer_path=WithProperties('%s/tools/clobberer/clobberer.py', 'builddir'),
            flunkOnFailure=False,
            haltOnFailure=False,
            timeout=3600, # One hour, because Windows is slow
        )
        #Clobber support end


        mozilla2_dep_factory.addStep(Mercurial(
            branchType='inrepo',
            mode='update',
            repourl="%s%s" % (HGURL, hg_branch),
            alwaysUseLatest=True,
            timeout=60*60,
            retry=[60,3],
            clobberOnBranchChange=False,
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, hg_branch)
        changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['python', 'client.py' ] + branch['client_py_args'] + ['checkout'],
            description=['running', 'client.py', 'checkout'],
            descriptionDone=['python', 'client.py', 'checkout'],
            haltOnFailure=True,
            timeout=60*60,
        ))
        mozilla2_dep_factory.addStep(GetHgRevision(
            workdir='build/mozilla'
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, mozilla_branch)

        link_tag = 'moz'
        if mozilla_branch == 'mozilla-central':
            link_tag = 'm-c'
        changesetLink += '/%(hg_revision)s title="Built from Mozilla revision %(hg_revision)s">' + link_tag + ':%(hg_revision)s</a>'

        mozilla2_dep_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['rm', '-rfv', 'configs'],
            description=['removing', 'configs'],
            descriptionDone=['remove', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['hg', 'clone', CONFIG_REPO_URL, 'configs'],
            description=['checking', 'out', 'configs'],
            descriptionDone=['checkout', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            # cp configs/$subdir/$platform/$branchName/mozconfig .mozconfig
            command=['cp', 'configs/%s/%s/%s/%s' % (CONFIG_SUBDIR,
                                                 platform,
                                                 branch_name,
                                                 mozconfig
                                                 ),
                     '.mozconfig'],
            description=['copying', 'mozconfig'],
            descriptionDone=['copy', 'mozconfig'],
            haltOnFailure=True
        ))
        mozilla2_dep_factory.addStep(ShellCommand(
            command=['cat', '.mozconfig'],
        ))

        buildcmd = 'build'
        mozilla2_dep_factory.addStep(Compile(
            command=['make', '-f', 'client.mk', buildcmd],
            env=pf['env'],
            haltOnFailure=True,
            timeout=60*60
        ))

        # TODO change to calendar, when we support leak tests
        if branch.get('leak'):
            addLeakTestSteps(mozilla2_dep_factory, branch, pf, platform, 'thunderbird', logUploadDir, DEFAULTS)

        if platform.find('debug') == -1:
            if branch['package']:
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['make', 'package'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=True
                    ))
                    if platform.startswith("win32"):
                        mozilla2_dep_factory.addStep(ShellCommand(
                            command=['make', 'installer'],
                            workdir='build/%s' % pf['platform_objdir'],
                            haltOnFailure=True
                        ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['make', 'package-compare'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=False
                    ))
            mozilla2_dep_factory.addStep(SetMozillaBuildProperties(
                objdir='build/%s/mozilla' % pf['platform_objdir']
            ))
            if upload_stage:
                mozilla2_dep_factory.addStep(MozillaStageUpload(
                    objdir='%s/mozilla' % pf['platform_objdir'],
                    username=branch.get('stage_username',STAGE_USERNAME),
                    milestone=milestone,
                    remoteHost=branch.get('stage_server',STAGE_SERVER),
                    remoteBasePath=stage_base_path,
                    packageGlob=upload_glob,
                    platform=realPlatform,
                    group=branch.get('stage_group',STAGE_GROUP),
                    sshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    releaseToLatest=False,
                    releaseToDated=False,
                    releaseToTinderboxBuilds=True,
                    tinderboxBuildsDir='%s-%s' % (milestone, platform),
                    dependToDated=True
                ))
            # XXX Hack to allow calendar to run xpcshell-tests
            if platform.find("macosx") == -1:
                mozilla2_dep_factory.addStep(unittest_steps.MozillaCheck,
                                             test_name="xpcshell-tests",
                                             warnOnWarnings=True,
                                             workdir="build/%s/calendar/test" % pf['platform_objdir'],
                                             timeout=5*60, # 5 minutes.
                                             )
            if codesighs:
                if platform.find('win32') == -1 and platform.find('linux64') == -1:
                    # Codesighs
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['make'],
                        workdir='build/%s/mozilla/tools/codesighs' % pf['platform_objdir']
                    ))
                    # TODO change to calendar when we support codesighs
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['wget', '-O', 'codesize-auto-old.log',
                         'http://%s/pub/mozilla.org/thunderbird/%s/codesize-auto.log' %\
                          (branch.get('stage_server',STAGE_SERVER), logUploadDir)],
                        env=pf['env']
                    ))
                    mozilla2_dep_factory.addStep(Codesighs(
                        objdir='../%s/mozilla' % pf['platform_objdir'],
                        platform=realPlatform,
                        workdir='build/mozilla',
                        env=pf['env']
                    ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['cat', 'codesize-auto-diff.log']
                    ))
                    mozilla2_dep_factory.addStep(ShellCommand(
                        command=['scp', '-o', 'User=%s' % branch.get('stage_username',STAGE_USERNAME),
                         '-o', 'IdentityFile=~/.ssh/%s' % branch.get('stage_ssh_key',STAGE_SSH_KEY),
                         'codesize-auto.log',
                         '%s:%s/%s' % (branch.get('stage_server',STAGE_SERVER), stage_base_path, logUploadDir)]
                    ))

        mozilla2_dep_builder = {
            'name': '%s %s' % (pf['base_name'], builder_type),
            'slavenames': pf['slaves'],
            'builddir': '%s-%s' % (name, platform),
            'factory': mozilla2_dep_factory,
            'category': name,
            'locks' : [lock],
        }
        c['builders'].append(mozilla2_dep_builder)

        lockname = 'lock-%s-%s-nightly' % (name, platform)
        lock = locks.SlaveLock(lockname)

        mozilla2_nightly_factory = factory.BuildFactory()
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['echo', WithProperties('Building on: %(slavename)s')],
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
          command=['echo', 'TinderboxPrint:', WithProperties('s: %(slavename)s')]
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['find', '.', '-maxdepth', '2', '-mtime', '+7', '-exec',
                     'rm', '-rfv', '{}', ';'],
            env=pf['env'],
            workdir='.',
            description=['cleanup', 'old', 'symbols'],
            flunkOnFailure=False,
        ))

        mozilla2_nightly_factory.addStep(SetProperty,
         name='basename_pwd',
         command=['bash', '-c', 'basename "$PWD"'],
         property='builddir',
         workdir='.'
        )

        mozilla2_nightly_factory.addStep(ShellCommand,
         command=['rm', '-rf', 'tools'],
         description=['clobber', 'build tools'],
         workdir='.'
        )
        mozilla2_nightly_factory.addStep(ShellCommand,
         command=['bash', '-c',
          'if [ ! -d tools ]; then hg clone %s%s; fi' % (HGURL, build_tools_repo)],
         description=['clone', 'build tools'],
         workdir='.',
         flunkOnFailure=False,
         haltOnFailure=False,
        )

        if build_space > 0:
            command = ['python', 'tools/buildfarm/maintenance/purge_builds.py',
                 '-s', str(build_space)]

            # Ignore the current dir also.
            command.extend(["-n", WithProperties("%(builddir)s")])
            command.append("..")

            mozilla2_nightly_factory.addStep(ShellCommand,
             name='clean_old_builds',
             command=command,
             description=['cleaning', 'old', 'builds'],
             descriptionDone=['clean', 'old', 'builds'],
             haltOnFailure=True,
             workdir='.',
             timeout=3600, # One hour, because Windows is slow
            )

        mozilla2_nightly_factory.addStep(Mercurial(
            branchType='inrepo',
            mode='clobber',
            repourl="%s%s" % (HGURL, hg_branch),
            timeout=60*60,
            retry=[60,3],
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, hg_branch)
        changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        client_py_args = branch['client_py_args']
        if branch.get('nightly_client_py_args'):
            client_py_args = client_py_args + branch['nightly_client_py_args']
            import sys
            print >> sys.stderr, "client_py_args will be : %s" % client_py_args

        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['python', 'client.py'] + client_py_args + ['checkout'],
            description=['running', 'client.py', 'checkout'],
            descriptionDone=['client.py', 'checkout'],
            haltOnFailure=True,
            timeout=60*60,
        ))
        mozilla2_nightly_factory.addStep(GetHgRevision(
            workdir='build/mozilla'
        ))
        changesetLink = '<a href=%s%s/rev' % (HGURL, mozilla_branch)
        changesetLink += '/%(hg_revision)s title="Built from Mozilla revision %(hg_revision)s">moz:%(hg_revision)s</a>'
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['echo', 'TinderboxPrint:', WithProperties(changesetLink)]
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['rm', '-rfv', 'configs'],
            description=['removing', 'configs'],
            descriptionDone=['remove', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['hg', 'clone', CONFIG_REPO_URL, 'configs'],
            description=['checking', 'out', 'configs'],
            descriptionDone=['checkout', 'configs'],
            haltOnFailure=True
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            # cp configs/$subdir/$platform/mozconfig .mozconfig
            command=['cp', 'configs/%s/%s/%s/%s' % (CONFIG_SUBDIR,
                                                 platform,
                                                 branch_name,
                                                 mozconfig
                                                 ),
                     '.mozconfig'],
            description=['copying', 'mozconfig'],
            descriptionDone=['copy', 'mozconfig'],
            haltOnFailure=True
        ))
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['cat', '.mozconfig'],
        ))

        buildcmd = 'build'
        mozilla2_nightly_factory.addStep(Compile(
            command=['make', '-f', 'client.mk', buildcmd],
            env=pf['env'],
            haltOnFailure=True,
            timeout=60*60
        ))
        if pf['upload_symbols']:
            mozilla2_nightly_factory.addStep(ShellCommand(
                command=['make', 'buildsymbols'],
                env=pf['env'],
                workdir='build/%s/calendar/lightning' % pf['platform_objdir'],
                haltOnFailure=True,
                timeout=60*60,
            ))
        if branch['package']:
                mozilla2_nightly_factory.addStep(ShellCommand(
                    command=['make', 'package'],
                    workdir='build/%s' % pf['platform_objdir'],
                    haltOnFailure=True
                ))
                if platform.startswith("win32"):
                    mozilla2_nightly_factory.addStep(ShellCommand(
                        command=['make', 'installer'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=True
                    ))
                mozilla2_nightly_factory.addStep(ShellCommand(
                        command=['make', 'package-compare'],
                        workdir='build/%s' % pf['platform_objdir'],
                        haltOnFailure=False
                    ))
                mozilla2_nightly_factory.addStep(ShellCommand(
                    command=['make', '-C',
                             '%s/mozilla/tools/update-packaging' % pf['platform_objdir']],
                    description=['create', 'complete', 'update'],
                    haltOnFailure=True
                ))
        mozilla2_nightly_factory.addStep(SetMozillaBuildProperties(
                objdir='build/%s/mozilla' % pf['platform_objdir']
        ))
        if upload_stage:
            mozilla2_nightly_factory.addStep(MozillaStageUpload(
                    objdir='%s/mozilla' % pf['platform_objdir'],
                    username=branch.get('stage_username',STAGE_USERNAME),
                    milestone=milestone,
                    remoteHost=branch.get('stage_server',STAGE_SERVER),
                    remoteBasePath=stage_base_path,
                    packageGlob=upload_glob,
                    platform=realPlatform,
                    group=branch.get('stage_group',STAGE_GROUP),
                    sshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    releaseToDated=True,
                    releaseToLatest=True,
                    releaseToTinderboxBuilds=True,
                    uploadCompleteMar=branch.get('upload_complete_mar', True),
                    tinderboxBuildsDir='%s-%s' % (milestone, platform),
                    dependToDated=True
            ))

        create_snippet = pf.get('create_snippet', branch['create_snippet'])
        aus = getConfig(DEFAULTS, branch, 'aus')

        if create_snippet:
            mozilla2_nightly_factory.addStep(CreateCompleteUpdateSnippet(
                objdir='build/%s/mozilla' % pf['platform_objdir'],
                milestone=milestone,
                baseurl='%s/nightly' % download_base_url,
            ))
	    # New style aus configuration
            if aus:
                uploadUpdateSnippet(mozilla2_nightly_factory, aus, pf)
    
        if pf['upload_symbols']:
            mozilla2_nightly_factory.addStep(ShellCommand(
                command=['make', 'uploadsymbols'],
                env=pf['env'],
                workdir='build/%s/calendar/lightning' % pf['platform_objdir'],
                haltOnFailure=True
            ))
        if l10n:
           l10n_scheduler="%s %s %s l10n nightly scheduler" % (product_name, platform, branch_name)
           mozilla2_nightly_factory.addStep(trigger.Trigger(
               schedulerNames=[l10n_scheduler],
               waitForFinish=False,
               flunkOnFailure=False,
               haltOnFailure=False,
            ))

        # XXX Ugh, why do we have to do this in the middle of so many other steps?
        if 'extensions' in branch:
            l10n_ext_schedulers = []
            for extensionName in branch['extensions'].keys():
                extension = branch['extensions'][extensionName]
                l10n_ext_schedulers.append('%s %s l10n extension nightly scheduler' % (platform, branch_name))

            mozilla2_nightly_factory.addStep(trigger.Trigger(
               name='trigger_extension_repacks',
               schedulerNames=l10n_ext_schedulers,
               waitForFinish=False,
               flunkOnFailure=False,
               haltOnFailure=False,
            ))

                    
                

        # in order to save disk space we clobber 'build' at the end.
        # we don't want to clobber anything else though, because symbols
        # sit alongside
        mozilla2_nightly_factory.addStep(ShellCommand(
            command=['rm', '-rfv', 'build'],
            env=pf['env'],
            workdir='.'
        ))
        nightly_builder = pf.get('nightly_builder',  branch.get('nightly_builder', DEFAULTS.get('nightly_builder')))
        import sys
        nightly_slaves = pf['slaves']
        if nightly_builder:
            print >> sys.stderr, "%s nightly builder override: %s" % (pf['base_name'], nightly_builder)
            nightly_slaves = nightly_builder

        mozilla2_nightly_builder = {
            'name': '%s nightly' % pf['base_name'],
            'slavenames': nightly_slaves,
            'builddir': '%s-%s-nightly' % (name, platform),
            'factory': mozilla2_nightly_factory,
            'category': name,
            'locks': [lock],
        }
        if nightly:
            c['builders'].append(mozilla2_nightly_builder)

        if l10n:
            l10n_nightly_updates = pf.get('l10n_nightly_updates',  branch.get('l10n_nightly_updates', DEFAULTS.get('l10n_nightly_updates', name)))
            l10n_dated_dirs = False
            if l10n_nightly_updates:
                l10n_dated_dirs = True
            print >> sys.stderr, "branchName: %s, repoPath: %s" % (branch_name, hg_branch)
            mozilla2_l10n_repack_factory = CCNightlyRepackFactory(
                    nightly=True,
                    hgHost='hg.mozilla.org',
                    tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                    project=branch.get('stage_project',appname),
                    appName=product,
                    enUSBinaryURL='%s/nightly/latest-%s/' % (download_base_url, milestone),
                    downloadBaseURL=download_base_url,
                    stageServer=branch.get('stage_server',STAGE_SERVER),
                    stageUsername=branch.get('stage_username',STAGE_USERNAME),
                    stageSshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    ausBaseUploadDir=aus['base_upload_dir'],
                    ausUser=aus['user'],
                    ausHost=aus['host'],
                    updatePlatform=pf['update_platform'],
                    branchName=branch_name,
                    repoPath=hg_branch,
                    mozRepoPath=mozilla_branch,
                    l10nRepoPath=getConfig(DEFAULTS, branch, 'l10n_repo'),
                    l10nNightlyUpdate=l10n_nightly_updates,
                    l10nDatedDirs=l10n_dated_dirs,
                    buildToolsRepoPath='build/tools',
                    compareLocalesRepoPath='build/compare-locales',
                    compareLocalesTag='RELEASE_AUTOMATION',
                    buildSpace=build_space,
                    clobberURL=clobber_url,
                    clobberTime=30*24*7, #Hours before clobber
            )

            mozilla2_l10n_repack_builder = {
                'name': '%s %s l10n' % (product_name, pf['base_name']),
                'slavenames': pf['slaves'],
                'builddir': '%s-%s-l10n-full' % (name, platform),
                'factory': mozilla2_l10n_repack_factory,
                'category': '%s-l10n' % name,
            }
            c['builders'].append(mozilla2_l10n_repack_builder)
            c['schedulers'].append(NightlyL10n(
                                   name='%s %s %s l10n nightly scheduler' % (product_name, platform, branch_name),
                                   platform=realPlatform,
                                   tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                                   hour=[7],
                                   builderNames=[ '%s %s l10n' % (product_name, pf['base_name']) ],
                                   repoType='hg',
                                   branch=hg_branch,
                                   baseTag='default',
                                   localesFile='%s/locales/all-locales' % product,
                                  )) 

            mozilla2_l10n_repack_dep_factory = CCNightlyRepackFactory(
                    hgHost='hg.mozilla.org',
                    tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                    project=branch.get('stage_project',appname),
                    appName=product,
                    nightly=False, 
                    enUSBinaryURL='%s/nightly/latest-%s/' % (download_base_url, milestone), 
                    stageServer=branch.get('stage_server',STAGE_SERVER),
                    stageUsername=branch.get('stage_username',STAGE_USERNAME),
                    stageSshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                    branchName=branch_name,
                    repoPath=hg_branch,
                    mozRepoPath=mozilla_branch,
                    l10nRepoPath=getConfig(DEFAULTS, branch, 'l10n_repo'),
                    buildToolsRepoPath='build/tools',
                    compareLocalesRepoPath='build/compare-locales',
                    compareLocalesTag='RELEASE_AUTOMATION',
                    buildSpace=build_space,
                    clobberURL=clobber_url,
                    clobberTime=30*24*7, #Hours before clobber
            )
            mozilla2_l10n_repack_dep_builder = {
                'name': '%s %s l10n build' % (product_name, pf['base_name']),
                'slavenames': pf['slaves'],
                'builddir': '%s-%s-l10n-dep' % (name, platform),
                'factory': mozilla2_l10n_repack_dep_factory,
                'category': '%s-l10n' % name,
            }
            c['builders'].append(mozilla2_l10n_repack_dep_builder)

        if 'extensions' in branch:
            for extensionName in branch['extensions'].keys():
                extension = branch['extensions'][extensionName]
                ext_base_url = getExtensionConfig(extension, DEFAULTS, 'download_base_url')

                if extension.get('l10n'):
                    binaryURL = extension.get('enUSBinaryURL',
                                              '%s/nightly/latest-%s/' %
                                                (ext_base_url, milestone))
                    ext_repack_factory = CCNightlyExtensionRepackFactory(
                        hgHost='hg.mozilla.org',
                        tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                        project=branch.get('stage_project',appname),
                        appName=appname,
                        extensionName=extensionName,
                        extensionPath=extension.get('subdir'),
                        enUSBinaryURL=binaryURL, 
                        stageServer=branch.get('stage_server',STAGE_SERVER),
                        stageUsername=branch.get('stage_username',STAGE_USERNAME),
                        stageSshKey=branch.get('stage_ssh_key',STAGE_SSH_KEY),
                        branchName=branch_name,
                        configRepoPath=CONFIG_REPO_PATH,
                        configSubDir=appname,
                        platform=platform,
                        mozconfig='%s/%s/%s' % (platform,
                                                branch_name,
                                                 mozconfig),
                        env={},
                        repoPath=hg_branch,
                        mozRepoPath=mozilla_branch,
                        l10nRepoPath=getConfig(DEFAULTS, branch, 'l10n_repo'),
                        buildToolsRepoPath='build/tools',
                        compareLocalesRepoPath='build/compare-locales',
                        compareLocalesTag='RELEASE_AUTOMATION',
                        buildSpace=build_space,
                        clobberURL=clobber_url,
                        clobberTime=30*24*7, #Hours before clobber
                    )   
                    c['builders'].append({
                        'name': '%s l10n extension build' % pf['base_name'],
                        'slavenames': pf['slaves'],
                        'builddir': '%s-%s-l10n-ext' % (name, platform),
                        'factory': ext_repack_factory,
                        'category': '%s-l10n' % name,
                    })  
                c['schedulers'].append(NightlyL10n(
                                   name='%s %s l10n extension nightly scheduler' % (platform, branch_name),
                                   platform=realPlatform,
                                   tree=getConfig(DEFAULTS, branch, 'l10n_tree'),
                                   builderNames=['%s l10n extension build' % pf['base_name']],
                                   repoType='hg',
                                   branch=hg_branch,
                                   baseTag='default',
                                   localesFile='%s/locales/all-locales' % product,
                                  ))

         



####### STATUS TARGETS

from buildbot.status import html

baseport=8031

c['status'].append(html.WebStatus(
    http_port=baseport, allowForce=True,
))

c['status'].append(html.WebStatus(
    http_port=8032, allowForce=False,
))

####### PROJECT IDENTITY

# the 'projectName' string will be used to describe the project that this
# buildbot is working on. For example, it is used as the title of the
# waterfall HTML page. The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.

c['projectName'] = "Calendar"
c['buildbotURL'] = "http://build.mozillamessaging.com/buildbot/calendar"


#XXX
#c['schedulers'] = []

def mergeRequests(builder, req1, req2):
    import copy
    r1 = copy.copy(req1.source)
    r2 = copy.copy(req2.source)
    import sys
    print >> sys.stderr, "XXX: mergeRequests(%s,%s,%s)" % (builder, r1, r2)
    print >> sys.stderr, "XXX: r1.branch=%s r2.branch=%s" % (r1.branch, r2.branch)
 #   r1.branch = r2.branch 
 #   print >> sys.stderr, "XXX: making both branches be %s" % r1.branch
    rv = r1.canBeMergedWith(r2)
    print >> sys.stderr, "XXX: mergeRequests rv=%s" % rv
    #XXX: Busted, buildbot re-checks canBeMergedWith and asserts, so can't merge effectively
    #XXX: what buildbot thinks are non-mergeable requests, grr.
    return rv

##c['mergeRequests'] = mergeRequests

 # Give certain builders priority over other builders                                                                                                    
def prioritizeBuilders(botmaster, builders):
    def sortkey(builder):
        priority = 0

        #Release above everything else
        if builder.builder_status.category == 'release':
            priority += 4

        #Nightly above most
        if builder.name.find('nightly') > 1:
            priority += 2

        return -priority, builder.getOldestRequestTime()

    builders.sort(key=sortkey)

    return builders
c['prioritizeBuilders'] = prioritizeBuilders