process/factory.py
author Edmund Wong <ewong@pw-wspx.org>
Thu, 21 Aug 2014 11:30:54 +0800
branchseamonkey-production
changeset 3802 1d60e4f2efa86e74b0df9f46a8c45360178046c9
parent 3801 fc8352b5740e2df7d7181b8bbcdd82272d893053
child 4397 eadcb80754900fd32ef8b6cefdee3fabe6b2bc72
permissions -rw-r--r--
Bug 840427 - Buildbotcustom changes for mock. (bug fixes) r=Callek

from __future__ import absolute_import

from datetime import datetime
import os.path, re
import urllib
import random

from twisted.python import log

from buildbot.process.buildstep import regex_log_evaluator
from buildbot.process.factory import BuildFactory
from buildbot.steps.shell import WithProperties
from buildbot.steps.transfer import FileDownload, JSONPropertiesDownload, JSONStringDownload
from buildbot.steps.dummy import Dummy
from buildbot import locks
from buildbot.status.builder import worst_status

import buildbotcustom.common
import buildbotcustom.status.errors
import buildbotcustom.steps.base
import buildbotcustom.steps.misc
import buildbotcustom.steps.release
import buildbotcustom.steps.source
import buildbotcustom.steps.test
import buildbotcustom.steps.transfer
import buildbotcustom.steps.updates
import buildbotcustom.steps.talos
import buildbotcustom.steps.unittest
import buildbotcustom.steps.signing
import buildbotcustom.steps.mock
import buildbotcustom.env
import buildbotcustom.misc_scheduler
import build.paths
import release.info
import release.paths
reload(buildbotcustom.common)
reload(buildbotcustom.status.errors)
reload(buildbotcustom.steps.base)
reload(buildbotcustom.steps.misc)
reload(buildbotcustom.steps.release)
reload(buildbotcustom.steps.source)
reload(buildbotcustom.steps.test)
reload(buildbotcustom.steps.transfer)
reload(buildbotcustom.steps.updates)
reload(buildbotcustom.steps.talos)
reload(buildbotcustom.steps.unittest)
reload(buildbotcustom.steps.signing)
reload(buildbotcustom.steps.mock)
reload(buildbotcustom.env)
reload(build.paths)
reload(release.info)
reload(release.paths)

from buildbotcustom.status.errors import purge_error, global_errors, \
  upload_errors
from buildbotcustom.steps.base import ShellCommand, SetProperty, Mercurial, \
  Trigger, RetryingShellCommand, RetryingSetProperty
from buildbotcustom.steps.misc import TinderboxShellCommand, SendChangeStep, \
  GetBuildID, MozillaClobberer, FindFile, DownloadFile, UnpackFile, \
  SetBuildProperty, DisconnectStep, OutputStep, ScratchboxCommand, \
  RepackPartners, UnpackTest, FunctionalStep, setBuildIDProps, \
  RetryingScratchboxProperty
from buildbotcustom.steps.release import UpdateVerify, L10nVerifyMetaDiff, \
  SnippetComparison
from buildbotcustom.steps.source import MercurialCloneCommand
from buildbotcustom.steps.test import GraphServerPost
from buildbotcustom.steps.updates import CreateCompleteUpdateSnippet, \
  CreatePartialUpdateSnippet
from buildbotcustom.steps.signing import SigningServerAuthenication
from buildbotcustom.env import MozillaEnvironments
from buildbotcustom.common import getSupportedPlatforms, getPlatformFtpDir, \
  genBuildID, reallyShort
from buildbotcustom.steps.mock import MockReset, MockInit, MockCommand, MockInstall, \
  MockMozillaCheck, MockProperty, RetryingMockProperty, RetryingMockCommand, \
  MockAliveTest, MockCompareLeakLogs, MockScratchboxCommand, MockMozillaPackagedMochitests, \
  MockMozillaPackagedXPCShellTests, MockMozillaPackagedReftests

import buildbotcustom.steps.unittest as unittest_steps

import buildbotcustom.steps.talos as talos_steps
from buildbot.status.builder import SUCCESS, FAILURE

from release.paths import makeCandidatesDir

# limit the number of clones of the try repository so that we don't kill
# dm-vcview04 if the master is restarted, or there is a large number of pushes
hg_try_lock = locks.MasterLock("hg_try_lock", maxCount=20)

hg_l10n_lock = locks.MasterLock("hg_l10n_lock", maxCount=20)

# Limit ourselves to uploading 4 things at a time
upload_lock = locks.MasterLock("upload_lock", maxCount=4)

SIGNING_SERVER_CERT = os.path.join(
    os.path.dirname(build.paths.__file__),
    '../../../release/signing/host.cert')

class DummyFactory(BuildFactory):
    def __init__(self, delay=5, triggers=None):
        BuildFactory.__init__(self)
        self.addStep(Dummy(delay))
        if triggers:
            self.addStep(Trigger(
                schedulerNames=triggers,
                waitForFinish=False,
                ))

def makeDummyBuilder(name, slaves, category=None, delay=0, triggers=None, properties=None):
    properties = properties or {}
    builder = {
            'name': name,
            'factory': DummyFactory(delay, triggers),
            'builddir': name,
            'slavenames': slaves,
            'properties': properties,
            }
    if category:
        builder['category'] = category
    return builder

def postUploadCmdPrefix(upload_dir=None,
        branch=None,
        product=None,
        revision=None,
        version=None,
        who=None,
        builddir=None,
        buildid=None,
        buildNumber=None,
        to_tinderbox_dated=False,
        to_tinderbox_builds=False,
        to_dated=False,
        to_latest=False,
        to_try=False,
        to_shadow=False,
        to_candidates=False,
        to_mobile_candidates=False,
        nightly_dir=None,
        as_list=True,
        signed=False,
        ):
    """Returns a post_upload.py command line for the given arguments.

    If as_list is True (the default), the command line will be returned as a
    list of arguments.  Some arguments may be WithProperties instances.

    If as_list is False, the command will be returned as a WithProperties
    instance representing the entire command line as a single string.

    It is expected that the returned value is augmented with the list of files
    to upload, and where to upload it.
    """

    cmd = ["post_upload.py"]

    if upload_dir:
        cmd.extend(["--tinderbox-builds-dir", upload_dir])
    if branch:
        cmd.extend(["-b", branch])
    if product:
        cmd.extend(["-p", product])
    if buildid:
        cmd.extend(['-i', buildid])
    if buildNumber:
        cmd.extend(['-n', buildNumber])
    if version:
        cmd.extend(['-v', version])
    if revision:
        cmd.extend(['--revision', revision])
    if who:
        cmd.extend(['--who', who])
    if builddir:
        cmd.extend(['--builddir', builddir])
    if to_tinderbox_dated:
        cmd.append('--release-to-tinderbox-dated-builds')
    if to_tinderbox_builds:
        cmd.append('--release-to-tinderbox-builds')
    if to_try:
        cmd.append('--release-to-try-builds')
    if to_latest:
        cmd.append("--release-to-latest")
    if to_dated:
        cmd.append("--release-to-dated")
    if to_shadow:
        cmd.append("--release-to-shadow-central-builds")
    if to_candidates:
        cmd.append("--release-to-candidates-dir")
    if to_mobile_candidates:
        cmd.append("--release-to-mobile-candidates-dir")
    if nightly_dir:
        cmd.append("--nightly-dir=%s" % nightly_dir)
    if signed:
        cmd.append("--signed")

    if as_list:
        return cmd
    else:
        # Remove WithProperties instances and replace them with their fmtstring
        for i,a in enumerate(cmd):
            if isinstance(a, WithProperties):
                cmd[i] = a.fmtstring
        return WithProperties(' '.join(cmd))

def parse_make_upload(rc, stdout, stderr):
    ''' This function takes the output and return code from running
    the upload make target and returns a dictionary of important
    file urls.'''
    retval = {}
    for m in re.findall("^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm))",
                        "\n".join([stdout, stderr]), re.M):
        if 'devel' in m and m.endswith('.rpm'):
            retval['develRpmUrl'] = m
        elif 'tests' in m and m.endswith('.rpm'):
            retval['testsRpmUrl'] = m
        elif m.endswith('.rpm'):
            retval['packageRpmUrl'] = m
        elif m.endswith("crashreporter-symbols.zip"):
            retval['symbolsUrl'] = m
        elif m.endswith("tests.tar.bz2") or m.endswith("tests.zip"):
            retval['testsUrl'] = m
        elif m.endswith('apk') and 'unsigned-unaligned' in m:
            retval['unsignedApkUrl'] = m
        elif m.endswith('apk') and 'robocop' in m:
            retval['robocopApkUrl'] = m
        elif 'jsshell-' in m and m.endswith('.zip'):
            retval['jsshellUrl'] = m
        else:
            retval['packageUrl'] = m
    return retval

def short_hash(rc, stdout, stderr):
    ''' This function takes an hg changeset id and returns just the first 12 chars'''
    retval = {}
    retval['got_revision'] = stdout[:12]
    return retval

def get_signing_cmd(signingServers, python):
    if not python:
        python = 'python'
    cmd = [
        python,
        '%(toolsdir)s/release/signing/signtool.py',
        '--cachedir', '%(basedir)s/signing_cache',
        '-t', '%(basedir)s/token',
        '-n', '%(basedir)s/nonce',
        '-c', '%(toolsdir)s/release/signing/host.cert',
    ]
    for ss, user, passwd in signingServers:
        cmd.extend(['-H', ss])
    return ' '.join(cmd)

class MockMixin(object):
    warnOnFailure = True
    warnOnWarnings = True

    def addMockSteps(self):
        # do not add the steps more than once per instance
        if (hasattr(self, '_MockMixin_added_mock_steps')):
            return
        self._MockMixin_added_mock_steps = 1

        if self.mock_copyin_files:
            for source, target in self.mock_copyin_files:
                self.addStep(ShellCommand(
                    name='mock_copyin_%s' % source.replace('/', '_'),
                    command=['mock_mozilla', '-r', self.mock_target,
                             '--copyin', source, target],
                    haltOnFailure=True,
                ))
                self.addStep(MockCommand(
                    name='mock_chown_%s' % target.replace('/', '_'),
                    command='chown -R mock_mozilla %s' % target,
                    target=self.mock_target,
                    mock=True,
                    workdir='/',
                    mock_args=[],
                    mock_workdir_prefix=None,
                ))
        # This is needed for the builds to start
        self.addStep(MockCommand(
            name='mock_mkdir_basedir',
            command=WithProperties(
                "mkdir -p %(basedir)s" + "/%s" % self.baseWorkDir),
            target=self.mock_target,
            mock=True,
            workdir='/',
            mock_workdir_prefix=None,
        ))
        if self.use_mock and self.mock_packages:
            self.addStep(MockInstall(
                target=self.mock_target,
                packages=self.mock_packages,
            ))

class BootstrapFactory(BuildFactory):
    def __init__(self, automation_tag, logdir, bootstrap_config,
                 cvsroot="pserver:anonymous@cvs-mirror.mozilla.org",
                 cvsmodule="mozilla"):
        """
    @type  cvsroot: string
    @param cvsroot: The CVSROOT to use for checking out Bootstrap.

    @type  cvsmodule: string
    @param cvsmodule: The CVS module to use for checking out Bootstrap.

    @type  automation_tag: string
    @param automation_tag: The CVS Tag to use for checking out Bootstrap.

    @type  logdir: string
    @param logdir: The log directory for Bootstrap to use.
                   Note - will be created if it does not already exist.

    @type  bootstrap_config: string
    @param bootstrap_config: The location of the bootstrap.cfg file on the
                             slave. This will be copied to "bootstrap.cfg"
                             in the builddir on the slave.
        """
        BuildFactory.__init__(self)
        self.addStep(ShellCommand(
         name='rm_builddir',
         description='clean checkout',
         workdir='.',
         command=['rm', '-rf', 'build'],
         haltOnFailure=1))
        self.addStep(RetryingShellCommand(
         name='checkout',
         description='checkout',
         workdir='.',
         command=['cvs', '-d', cvsroot, 'co', '-r', automation_tag,
                  '-d', 'build', cvsmodule],
         haltOnFailure=1,
        ))
        self.addStep(ShellCommand(
         name='copy_bootstrap',
         description='copy bootstrap.cfg',
         command=['cp', bootstrap_config, 'bootstrap.cfg'],
         haltOnFailure=1,
        ))
        self.addStep(ShellCommand(
         name='echo_bootstrap',
         description='echo bootstrap.cfg',
         command=['cat', 'bootstrap.cfg'],
         haltOnFailure=1,
        ))
        self.addStep(ShellCommand(
         name='create_logdir',
         description='(re)create logs area',
         command=['bash', '-c', 'mkdir -p ' + logdir],
         haltOnFailure=1,
        ))

        self.addStep(ShellCommand(
         name='rm_old_logs',
         description='clean logs area',
         command=['bash', '-c', 'rm -rf ' + logdir + '/*.log'],
         haltOnFailure=1,
        ))
        self.addStep(ShellCommand(
         name='make_test',
         description='unit tests',
         command=['make', 'test'],
         haltOnFailure=1,
        ))

def getPlatformMinidumpPath(platform):
    platform_minidump_path = {
        'linux': WithProperties('%(toolsdir:-)s/breakpad/linux/minidump_stackwalk'),
        'linuxqt': WithProperties('%(toolsdir:-)s/breakpad/linux/minidump_stackwalk'),
        'linux64': WithProperties('%(toolsdir:-)s/breakpad/linux64/minidump_stackwalk'),
        'win32': WithProperties('%(toolsdir:-)s/breakpad/win32/minidump_stackwalk.exe'),
        'win64': WithProperties('%(toolsdir:-)s/breakpad/win64/minidump_stackwalk.exe'),
        'macosx': WithProperties('%(toolsdir:-)s/breakpad/osx/minidump_stackwalk'),
        'macosx64': WithProperties('%(toolsdir:-)s/breakpad/osx64/minidump_stackwalk'),
        # Android uses OSX because the Foopies are OSX.
        'linux-android': WithProperties('%(toolsdir:-)s/breakpad/osx/minidump_stackwalk'),
        'android': WithProperties('%(toolsdir:-)s/breakpad/osx/minidump_stackwalk'),
        'android-xul': WithProperties('%(toolsdir:-)s/breakpad/osx/minidump_stackwalk'),
        }
    return platform_minidump_path[platform]

class RequestSortingBuildFactory(BuildFactory):
    """Base class used for sorting build requests according to buildid.

    In most cases the buildid of the request is calculated at the time when the
    build is scheduled.  For tests, the buildid of the sendchange corresponds
    to the buildid of the build. Sorting the test requests by buildid allows us
    to order them according to the order in which the builds were scheduled.
    This avoids the problem where build A of revision 1 completes (and triggers
    tests) after build B of revision 2. Without the explicit sorting done here,
    the test requests would be sorted [r2, r1], and buildbot would choose the
    latest of the set to run.

    We sort according to the following criteria:
        * If the request looks like a rebuild, use the request's submission time
        * If the request or any of the changes contains a 'buildid' property,
          use the greatest of these property values
        * Otherwise use the request's submission time
    """
    def newBuild(self, requests):
        def sortkey(request):
            # Ignore any buildids if we're rebuilding
            # Catch things like "The web-page 'rebuild' ...", or self-serve
            # messages, "Rebuilt by ..."
            if 'rebuil' in request.reason.lower():
                return int(genBuildID(request.submittedAt))

            buildids = []

            props = [request.properties] + [c.properties for c in request.source.changes]

            for p in props:
                try:
                    buildids.append(int(p['buildid']))
                except:
                    pass

            if buildids:
                return max(buildids)
            return int(genBuildID(request.submittedAt))

        try:
            sorted_requests = sorted(requests, key=sortkey)
            return BuildFactory.newBuild(self, sorted_requests)
        except:
            # Something blew up!
            # Return the orginal list
            log.msg("Error sorting build requests")
            log.err()
            return BuildFactory.newBuild(self, requests)

class MozillaBuildFactory(RequestSortingBuildFactory, MockMixin):
    ignore_dirs = [ 'info', 'rel-*']

    def __init__(self, hgHost, repoPath, buildToolsRepoPath, buildSpace=0,
            clobberURL=None, clobberTime=None, buildsBeforeReboot=None,
            branchName=None, baseWorkDir='build', hashType='sha512',
            baseMirrorUrls=None, baseBundleUrls=None, signingServers=None,
            enableSigning=True, env={}, enable_pymake=False, use_mock=False,
            mock_target=None, mock_packages=None, mock_copyin_files=None, **kwargs):
        BuildFactory.__init__(self, **kwargs)

        if hgHost.endswith('/'):
            hgHost = hgHost.rstrip('/')
        self.hgHost = hgHost
        self.repoPath = repoPath
        self.buildToolsRepoPath = buildToolsRepoPath
        self.buildToolsRepo = self.getRepository(buildToolsRepoPath)
        self.buildSpace = buildSpace
        self.clobberURL = clobberURL
        self.clobberTime = clobberTime
        self.buildsBeforeReboot = buildsBeforeReboot
        self.baseWorkDir = baseWorkDir
        self.hashType = hashType
        self.baseMirrorUrls = baseMirrorUrls
        self.baseBundleUrls = baseBundleUrls
        self.signingServers = signingServers
        self.enableSigning = enableSigning
        self.env = env.copy()
        self.enable_pymake = enable_pymake
        self.use_mock = use_mock
        self.mock_target = mock_target
        self.mock_packages = mock_packages
        self.mock_copyin_files = mock_copyin_files

        self.repository = self.getRepository(repoPath)
        if branchName:
          self.branchName = branchName
        else:
          self.branchName = self.getRepoName(self.repository)

        if self.signingServers and self.enableSigning:
            self.signing_command = get_signing_cmd(
                self.signingServers, self.env.get('PYTHON26'))
            self.env['MOZ_SIGN_CMD'] = WithProperties(self.signing_command)

        if self.enable_pymake:
          self.makeCmd = ['python2.7', WithProperties("%(basedir)s/build/build/pymake/make.py")]
        else:
          self.makeCmd = ['make']

        self.addStep(OutputStep(
         name='get_buildername',
         data=WithProperties('Building on: %(slavename)s'),
        ))
        self.addStep(OutputStep(
         name='tinderboxprint_buildername',
         data=WithProperties('TinderboxPrint: s: %(slavename)s'),
        ))
        self.addStep(OutputStep(
         name='tinderboxsummarymessage_buildername',
         data=WithProperties('TinderboxSummaryMessage: s: %(slavename)s'),
        ))
        if self.use_mock:
           self.addStep(OutputStep(
             name='checking_for_mock',
             data=WithProperties('Mock : true'),
           ))
        else:
           self.addStep(OutputStep(
             name='checking_for_mock',
             data=WithProperties('Mock : false'),
           ))
        self.addInitialSteps()

    def addInitialSteps(self):
        self.addStep(SetProperty(
            name='set_basedir',
            command=['bash', '-c', 'pwd'],
            property='basedir',
            workdir='.',
        ))
        # We need the basename of the current working dir so we can
        # ignore that dir when purging builds later.
        self.addStep(SetProperty(
            name='set_builddir',
            command=['bash', '-c', 'basename "$PWD"'],
            property='builddir',
            workdir='.',
        ))
        # XX remove flunkOnFailure after bug 558430 is fixed
        self.addStep(ShellCommand(
         name='rm_buildtools',
         command=['rm', '-rf', 'tools'],
         description=['clobber', 'build tools'],
         workdir='.',
         flunkOnFailure=False,
        ))
        self.addStep(MercurialCloneCommand(
         name='clone_buildtools',
         command=['hg', 'clone', self.buildToolsRepo, 'tools'],
         description=['clone', 'build tools'],
         workdir='.',
         retry=False
        ))
        self.addStep(SetProperty(
            name='set_toolsdir',
            command=['bash', '-c', 'pwd'],
            property='toolsdir',
            workdir='tools',
            env=self.env,
        ))

        if self.clobberURL is not None:
            self.addStep(MozillaClobberer(
             name='checking_clobber_times',
             branch=self.branchName,
             clobber_url=self.clobberURL,
             clobberer_path=WithProperties('%(builddir)s/tools/clobberer/clobberer.py'),
             clobberTime=self.clobberTime
            ))

        if self.buildSpace > 0:
            command = ['python', 'tools/buildfarm/maintenance/purge_builds.py',
                 '-s', str(self.buildSpace)]

            for i in self.ignore_dirs:
                command.extend(["-n", i])

            # These are the base_dirs that get passed to purge_builds.py.
            # The scratchbox dir is only present on linux slaves, but since
            # not all classes that inherit from MozillaBuildFactory provide
            # a platform property we can use for limiting the base_dirs, it
            # is easier to include scratchbox by default and simply have
            # purge_builds.py skip the dir when it isn't present.
            command.extend(["..","/scratchbox/users/cltbld/home/cltbld/build"])

            def parse_purge_builds(rc, stdout, stderr):
                properties = {}
                for stream in (stdout, stderr):
                    m = re.search('unable to free (?P<size>[.\d]+) (?P<unit>\w+) ', stream, re.M)
                    if m:
                        properties['purge_target'] = '%s%s' % (m.group('size'), m.group('unit'))
                    m = None
                    m = re.search('space only (?P<size>[.\d]+) (?P<unit>\w+)', stream, re.M)
                    if m:
                        properties['purge_actual'] = '%s%s' % (m.group('size'), m.group('unit'))
                    m = None
                    m = re.search('(?P<size>[.\d]+) (?P<unit>\w+) of space available', stream, re.M)
                    if m:
                        properties['purge_actual'] = '%s%s' % (m.group('size'), m.group('unit'))
                if not properties.has_key('purge_target'):
                    properties['purge_target'] = '%sGB' % str(self.buildSpace)
                return properties

            self.addStep(SetProperty(
             name='clean_old_builds',
             command=command,
             description=['cleaning', 'old', 'builds'],
             descriptionDone=['clean', 'old', 'builds'],
             haltOnFailure=True,
             workdir='.',
             timeout=3600, # One hour, because Windows is slow
             extract_fn=parse_purge_builds,
             log_eval_func=lambda c,s: regex_log_evaluator(c, s, purge_error),
             #env=self.env, #XXX Breaks SeaMonkey Repacks on old system
            ))

        if self.use_mock:
           self.addStep(MockReset(
                target=self.mock_target,
           ))
           self.addStep(MockInit(
                target=self.mock_target,
           ))
           self.addMockSteps()

    def addPeriodicRebootSteps(self):
        def do_disconnect(cmd):
            try:
                if 'SCHEDULED REBOOT' in cmd.logs['stdio'].getText():
                    return True
            except:
                pass
            return False
        self.addStep(DisconnectStep(
         name='maybe_rebooting',
         command=['python', 'tools/buildfarm/maintenance/count_and_reboot.py',
                  '-f', '../reboot_count.txt',
                  '-n', str(self.buildsBeforeReboot),
                  '-z'],
         description=['maybe rebooting'],
         force_disconnect=do_disconnect,
         warnOnFailure=False,
         flunkOnFailure=False,
         alwaysRun=True,
         workdir='.'
        ))

    def getRepoName(self, repo):
        return repo.rstrip('/').split('/')[-1]

    def getRepository(self, repoPath, hgHost=None, push=False):
        assert repoPath
        for prefix in ('http://', 'ssh://'):
            if repoPath.startswith(prefix):
                return repoPath
        if repoPath.startswith('/'):
            repoPath = repoPath.lstrip('/')
        if not hgHost:
            hgHost = self.hgHost
        proto = 'ssh' if push else 'http'
        return '%s://%s/%s' % (proto, hgHost, repoPath)

    def getPackageFilename(self, platform, platform_variation):
        if 'android' in self.complete_platform:
            packageFilename = '*arm.apk' #the arm.apk is to avoid
                                         #unsigned/unaligned apks
        elif 'maemo' in self.complete_platform:
            packageFilename = '*.linux-*-arm.tar.*'
        elif platform.startswith("linux64"):
            packageFilename = '*.linux-x86_64.tar.bz2'
        elif platform.startswith("linux"):
            packageFilename = '*.linux-i686.tar.bz2'
        elif platform.startswith("macosx"):
            packageFilename = '*.dmg'
        elif platform.startswith("win32"):
            packageFilename = '*.win32.zip'
        elif platform.startswith("win64"):
            packageFilename = '*.win64-x86_64.zip'
        else:
            return False
        return packageFilename

    def parseFileSize(self, propertyName):
        def getSize(rv, stdout, stderr):
            stdout = stdout.strip()
            return {propertyName: stdout.split()[4]}
        return getSize

    def parseFileHash(self, propertyName):
        def getHash(rv, stdout, stderr):
            stdout = stdout.strip()
            return {propertyName: stdout.split(' ',2)[1]}
        return getHash

    def unsetFilepath(self, rv, stdout, stderr):
        return {'filepath': None}

    def addFilePropertiesSteps(self, filename, directory, fileType, 
                               doStepIf=True, maxDepth=1, haltOnFailure=False):
        self.addStep(FindFile(
            name='find_filepath',
            description=['find', 'filepath'],
            doStepIf=doStepIf,
            filename=filename,
            directory=directory,
            filetype='file',
            max_depth=maxDepth,
            property_name='filepath',
            workdir='.',
            haltOnFailure=haltOnFailure
        ))
        self.addStep(SetProperty(
            description=['set', fileType.lower(), 'filename'],
            doStepIf=doStepIf,
            command=['basename', WithProperties('%(filepath)s')],
            property=fileType+'Filename',
            workdir='.',
            name='set_'+fileType.lower()+'_filename',
            haltOnFailure=haltOnFailure
        ))
        self.addStep(SetProperty(
            description=['set', fileType.lower(), 'size',],
            doStepIf=doStepIf,
            command=['bash', '-c', 
                     WithProperties("ls -l %(filepath)s")],
            workdir='.',
            name='set_'+fileType.lower()+'_size',
            extract_fn = self.parseFileSize(propertyName=fileType+'Size'),
            haltOnFailure=haltOnFailure
        ))
        self.addStep(SetProperty(
            description=['set', fileType.lower(), 'hash',],
            doStepIf=doStepIf,
            command=['bash', '-c', 
                     WithProperties('openssl ' + 'dgst -' + self.hashType +
                                    ' %(filepath)s')],
            workdir='.',
            name='set_'+fileType.lower()+'_hash',
            extract_fn=self.parseFileHash(propertyName=fileType+'Hash'),
            haltOnFailure=haltOnFailure
        ))
        self.addStep(SetProperty(
            description=['unset', 'filepath',],
            doStepIf=doStepIf,
            name='unset_filepath',
            command='echo "filepath:"',
            workdir=directory,
            extract_fn = self.unsetFilepath,
        ))

    def makeHgtoolStep(self, name='hg_update', repo_url=None, wc=None,
            mirrors=None, bundles=None, env=None,
            clone_by_revision=False, rev=None, workdir='build',
            use_properties=True, locks=None):

        if not env:
            env = self.env

        if use_properties:
            env = env.copy()
            env['PROPERTIES_FILE'] = 'buildprops.json'

        cmd = ['python', WithProperties("%(toolsdir)s/buildfarm/utils/hgtool.py")]

        if clone_by_revision:
            cmd.append('--clone-by-revision')

        if mirrors is None and self.baseMirrorUrls:
            mirrors = ["%s/%s" % (url, self.repoPath) for url in self.baseMirrorUrls]

        if mirrors:
            for mirror in mirrors:
                cmd.extend(["--mirror", mirror])

        if bundles is None and self.baseBundleUrls:
            bundles = ["%s/%s.hg" % (url, self.getRepoName(self.repository)) for url in self.baseBundleUrls]

        if bundles:
            for bundle in bundles:
                cmd.extend(["--bundle", bundle])

        if not repo_url:
            repo_url = self.repository

        if rev:
            cmd.extend(["--rev", rev])

        cmd.append(repo_url)

        if wc:
            cmd.append(wc)

        if locks is None:
            locks = []

        return RetryingMockCommand(
            name=name,
            command=cmd,
            timeout=60*60,
            env=env,
            workdir=workdir,
            haltOnFailure=True,
            flunkOnFailure=True,
            locks=locks,
            mock=self.use_mock,
            target=self.mock_target,
        )

    def addGetTokenSteps(self):
        token = "token"
        nonce = "nonce"
        self.addStep(MockCommand(
            command=['rm', '-f', nonce],
            workdir='.',
            name='rm_nonce',
            description=['remove', 'old', 'nonce'],
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(SigningServerAuthenication(
            servers=self.signingServers,
            server_cert=SIGNING_SERVER_CERT,
            slavedest=token,
            workdir='.',
            name='download_token',
        ))


class MercurialBuildFactory(MozillaBuildFactory, MockMixin):
    def __init__(self, objdir, platform, configRepoPath, configSubDir,
                 profiledBuild, mozconfig, srcMozconfig=None,
                 use_scratchbox=False, productName=None,
                 scratchbox_target='FREMANTLE_ARMEL', android_signing=False,
                 buildRevision=None, stageServer=None, stageUsername=None,
                 stageGroup=None, stageSshKey=None, stageBasePath=None,
                 stageProduct=None, post_upload_include_platform=False,
                 ausBaseUploadDir=None, updatePlatform=None,
                 downloadBaseURL=None, ausUser=None, ausSshKey=None,
                 ausHost=None, nightly=False,
                 checkTest=False, valgrindCheck=False,
                 graphServer=None, graphSelector=None, graphBranch=None,
                 baseName=None, uploadPackages=True, uploadSymbols=True,
                 createSnippet=False, createPartial=False, doCleanup=True,
                 packageSDK=False, packageTests=False, mozillaDir=None,
                 enable_ccache=False, stageLogBaseUrl=None,
                 triggeredSchedulers=None, triggerBuilds=False,
                 mozconfigBranch="production", useSharedCheckouts=False,
                 stagePlatform=None, testPrettyNames=False, l10nCheckTest=False,
                 doBuildAnalysis=False,
                 downloadSubdir=None,
                 multiLocale=False,
                 multiLocaleMerge=True,
                 compareLocalesRepoPath=None,
                 compareLocalesTag='RELEASE_AUTOMATION',
                 mozharnessRepoPath=None,
                 mozharnessTag='default',
                 multiLocaleScript=None,
                 multiLocaleConfig=None,
                 mozharnessMultiOptions=None,
                 tooltool_manifest_src=None,
                 tooltool_bootstrap="setup.sh",
                 tooltool_url_list=None,
                 tooltool_script='/tools/tooltool.py',
                 use_mock=None,
                 mock_target=None,
                 mock_packages=None,
                 mock_copyin_files=None,
                 **kwargs):
        MozillaBuildFactory.__init__(self, use_mock=use_mock, mock_target=mock_target, mock_packages=mock_packages,
                 mock_copyin_files=mock_copyin_files, **kwargs)

        # Make sure we have a buildid and builduid
        self.addStep(FunctionalStep(
         name='set_buildids',
         func=setBuildIDProps,
        ))

        self.objdir = objdir
        self.platform = platform
        self.configRepoPath = configRepoPath
        self.configSubDir = configSubDir
        self.profiledBuild = profiledBuild
        self.mozconfig = mozconfig
        self.srcMozconfig = srcMozconfig
        self.productName = productName
        self.buildRevision = buildRevision
        self.stageServer = stageServer
        self.stageUsername = stageUsername
        self.stageGroup = stageGroup
        self.stageSshKey = stageSshKey
        self.stageBasePath = stageBasePath
        if stageBasePath and not stageProduct:
            self.stageProduct = productName
        else:
            self.stageProduct = stageProduct
        self.doBuildAnalysis = doBuildAnalysis
        self.ausBaseUploadDir = ausBaseUploadDir
        self.updatePlatform = updatePlatform
        self.downloadBaseURL = downloadBaseURL
        self.downloadSubdir = downloadSubdir
        self.ausUser = ausUser
        self.ausSshKey = ausSshKey
        self.ausHost = ausHost
        self.nightly = nightly
        self.checkTest = checkTest
        self.valgrindCheck = valgrindCheck
        self.graphServer = graphServer
        self.graphSelector = graphSelector
        self.graphBranch = graphBranch
        self.baseName = baseName
        self.uploadPackages = uploadPackages
        self.uploadSymbols = uploadSymbols
        self.createSnippet = createSnippet
        self.createPartial = createPartial
        self.doCleanup = doCleanup
        self.packageSDK = packageSDK
        self.packageTests = packageTests
        self.enable_ccache = enable_ccache
        if self.enable_ccache:
            self.env['CCACHE_BASEDIR'] = WithProperties('%(basedir:-)s')
        self.triggeredSchedulers = triggeredSchedulers
        self.triggerBuilds = triggerBuilds
        self.mozconfigBranch = mozconfigBranch
        self.use_scratchbox = use_scratchbox
        self.scratchbox_target = scratchbox_target
        self.android_signing = android_signing
        self.post_upload_include_platform = post_upload_include_platform
        self.useSharedCheckouts = useSharedCheckouts
        self.testPrettyNames = testPrettyNames
        self.l10nCheckTest = l10nCheckTest
        self.tooltool_manifest_src = tooltool_manifest_src
        self.tooltool_url_list = tooltool_url_list or []
        self.tooltool_script = tooltool_script
        self.tooltool_bootstrap = tooltool_bootstrap
        self.use_mock = use_mock
        self.mock_target = mock_target
        self.mock_packages = mock_packages
        self.mock_copyin_files = mock_copyin_files

        assert len(self.tooltool_url_list) <= 1, "multiple urls not currently supported by tooltool"

        if self.uploadPackages:
            assert productName and stageServer and stageUsername
            assert stageBasePath
        if self.createSnippet:
            if 'android' in platform: #happens before we trim platform
                assert downloadSubdir
            assert ausBaseUploadDir and updatePlatform and downloadBaseURL
            assert ausUser and ausSshKey and ausHost

            # To preserve existing behavior, we need to set the 
            # ausFullUploadDir differently for when we are create all the
            # mars (complete+partial) ourselves. 
            if self.createPartial:
                # e.g.:
                # /opt/aus2/incoming/2/Firefox/mozilla-central/WINNT_x86-msvc
                self.ausFullUploadDir = '%s/%s' % (self.ausBaseUploadDir,
                                                   self.updatePlatform)
            else:
                # this is a tad ugly because we need python interpolation
                # as well as WithProperties, e.g.:
                # /opt/aus2/build/0/Firefox/mozilla-central/WINNT_x86-msvc/2008010103/en-US
                self.ausFullUploadDir = '%s/%s/%%(buildid)s/en-US' % \
                                          (self.ausBaseUploadDir, 
                                           self.updatePlatform)

        self.complete_platform = self.platform
        # we don't need the extra cruft in 'platform' anymore
        self.platform = platform.split('-')[0]
        self.stagePlatform = stagePlatform
        # it turns out that the cruft is useful for dealing with multiple types
        # of builds that are all done using the same self.platform.
        # Examples of what happens:
        #   platform = 'linux' sets self.platform_variation to []
        #   platform = 'linux-opt' sets self.platform_variation to ['opt']
        #   platform = 'linux-opt-rpm' sets self.platform_variation to ['opt','rpm']
        platform_chunks = self.complete_platform.split('-', 1)
        if len(platform_chunks) > 1:
                self.platform_variation = platform_chunks[1].split('-')
        else:
                self.platform_variation = []

        assert self.platform in getSupportedPlatforms()

        if self.graphServer is not None:
            self.tbPrint = False
        else:
            self.tbPrint = True

        # SeaMonkey/Thunderbird make use of mozillaDir. Firefox does not.
        if mozillaDir:
            self.mozillaDir = '/%s' % mozillaDir
            self.mozillaObjdir = '%s%s' % (self.objdir, self.mozillaDir)
        else:
            self.mozillaDir = ''
            self.mozillaObjdir = self.objdir

        # These following variables are useful for sharing build steps (e.g.
        # update generation) with subclasses that don't use object dirs (e.g.
        # l10n repacks).
        # 
        # We also concatenate the baseWorkDir at the outset to avoid having to
        # do that everywhere.
        self.mozillaSrcDir = '.%s' % self.mozillaDir
        self.absMozillaSrcDir = '%s%s' % (self.baseWorkDir, self.mozillaDir)
        self.absMozillaObjDir = '%s/%s' % (self.baseWorkDir, self.mozillaObjdir)

        self.latestDir = '/pub/mozilla.org/%s' % self.stageProduct + \
                         '/nightly/latest-%s' % self.branchName
        if self.post_upload_include_platform:
            self.latestDir += '-%s' % self.stagePlatform

        self.stageLogBaseUrl = stageLogBaseUrl
        if self.stageLogBaseUrl:
            # yes, the branchName is needed twice here so that log uploads work for all
            self.logUploadDir = '%s/%s-%s/' % (self.branchName, self.branchName,
                                               self.stagePlatform)
            self.logBaseUrl = '%s/%s' % (self.stageLogBaseUrl, self.logUploadDir)
        else:
            self.logUploadDir = 'tinderbox-builds/%s-%s/' % (self.branchName,
                                                             self.stagePlatform)
            self.logBaseUrl = 'http://%s/pub/mozilla.org/%s/%s' % \
                        ( self.stageServer, self.stageProduct, self.logUploadDir)

        # Need to override toolsdir as set by MozillaBuildFactory because
        # we need Windows-style paths.
        if self.platform.startswith('win'):
            self.addStep(SetProperty(
                command=['bash', '-c', 'pwd -W'],
                property='toolsdir',
                workdir='tools'
            ))
            self.addStep(SetProperty(
                command=['bash', '-c', 'pwd -W'],
                property='basedir',
                workdir='.'
            ))
        if self.use_scratchbox:
            self.addStep(ScratchboxCommand(
                command=["sb-conf", "select", self.scratchbox_target],
                name='set_target',
                env=self.env,
                sb=True,
                workdir='/',
            ))

        if self.use_mock:
            self.addMockSteps()

        if self.enable_ccache:
            self.addStep(ShellCommand(command=['ccache', '-z'],
                     name="clear_ccache_stats", warnOnFailure=False,
                     flunkOnFailure=False, haltOnFailure=False, env=self.env))
        if multiLocale:
            assert compareLocalesRepoPath and compareLocalesTag
            assert mozharnessRepoPath and mozharnessTag
            assert multiLocaleScript and multiLocaleConfig
            self.compareLocalesRepoPath = compareLocalesRepoPath
            self.compareLocalesTag = compareLocalesTag
            self.mozharnessRepoPath = mozharnessRepoPath
            self.mozharnessTag = mozharnessTag
            self.multiLocaleScript = multiLocaleScript
            self.multiLocaleConfig = multiLocaleConfig
            self.multiLocaleMerge = multiLocaleMerge
            if mozharnessMultiOptions:
                self.mozharnessMultiOptions = mozharnessMultiOptions
            else:
                self.mozharnessMultiOptions = ['--only-pull-locale-source',
                                               '--only-add-locales',
                                               '--only-package-multi']

            self.addMultiLocaleRepoSteps()

        self.multiLocale = multiLocale

        self.addBuildSteps()
        if self.uploadSymbols or self.packageTests:
            self.addBuildSymbolsStep()
        if self.uploadSymbols:
            self.addUploadSymbolsStep()
        if self.uploadPackages:
            self.addUploadSteps()
        if self.testPrettyNames:
            self.addTestPrettyNamesSteps()
        if self.l10nCheckTest:
            self.addL10nCheckTestSteps()
        if self.checkTest:
            self.addCheckTestSteps()
        if self.valgrindCheck:
            self.addValgrindCheckSteps()
        if self.createSnippet:
            self.addUpdateSteps()
        if self.triggerBuilds:
            self.addTriggeredBuildsSteps()
        if self.doCleanup:
            self.addPostBuildCleanupSteps()
        if self.enable_ccache:
            self.addStep(ShellCommand(command=['ccache', '-s'],
                     name="print_ccache_stats", warnOnFailure=False,
                     flunkOnFailure=False, haltOnFailure=False, env=self.env))
        if self.buildsBeforeReboot and self.buildsBeforeReboot > 0:
            self.addPeriodicRebootSteps()

    def addMultiLocaleRepoSteps(self):
        for repo,tag in ((self.compareLocalesRepoPath,self.compareLocalesTag),
                            (self.mozharnessRepoPath,self.mozharnessTag)):
            name=repo.rstrip('/').split('/')[-1]
            self.addStep(MockCommand(
                name='rm_%s'%name,
                command=['rm', '-rf', '%s' % name],
                description=['removing', name],
                descriptionDone=['remove', name],
                haltOnFailure=True,
                workdir='.',
                mock=self.use_mock,
                target=self.mock_target,
            ))
            self.addStep(MercurialCloneCommand(
                name='hg_clone_%s' % name,
                command=['hg', 'clone', self.getRepository(repo), name],
                description=['checking', 'out', name],
                descriptionDone=['checkout', name],
                haltOnFailure=True,
                workdir='.',
            ))
            self.addStep(MockCommand(
                name='hg_update_%s'% name,
                command=['hg', 'update', '-r', tag],
                description=['updating', name, 'to', tag],
                workdir=name,
                haltOnFailure=True,
                mock=self.use_mock,
                target=self.mock_target,
           ))


    def addTriggeredBuildsSteps(self,
                                triggeredSchedulers=None):
        '''Trigger other schedulers.
        We don't include these steps by default because different
        children may want to trigger builds at different stages.

        If triggeredSchedulers is None, then the schedulers listed in
        self.triggeredSchedulers will be triggered.
        '''
        if triggeredSchedulers is None:
            if self.triggeredSchedulers is None:
                return True
            triggeredSchedulers = self.triggeredSchedulers

        for triggeredScheduler in triggeredSchedulers:
            self.addStep(Trigger(
                schedulerNames=[triggeredScheduler],
                copy_properties=['buildid', 'builduid'],
                waitForFinish=False))

    def addBuildSteps(self):
        self.addPreBuildSteps()
        self.addSourceSteps()
        self.addConfigSteps()
        self.addDoBuildSteps()
        if self.signingServers and self.enableSigning:
            self.addGetTokenSteps()
        if self.doBuildAnalysis:
            self.addBuildAnalysisSteps()

    def addPreBuildSteps(self):
        if self.nightly:
            self.addStep(MockCommand(
             name='rm_builddir',
             command=['rm', '-rf', 'build'],
             env=self.env,
             workdir='.',
             timeout=60*60, # 1 hour
             mock=self.use_mock,
             target=self.mock_target
            ))
        pkg_patterns = []
        for product in ('firefox-', 'fennec', 'seamonkey'):
            pkg_patterns.append('%s/dist/%s*' % (self.mozillaObjdir,
                                                  product))

        self.addStep(ShellCommand(
         name='rm_old_pkg',
         command="rm -rf %s %s/dist/install/sea/*.exe " %
                  (' '.join(pkg_patterns), self.mozillaObjdir),
         env=self.env,
         description=['deleting', 'old', 'package'],
         descriptionDone=['delete', 'old', 'package'],
        ))
        if self.nightly:
            self.addStep(MockCommand(
             name='rm_old_symbols',
             command="find 20* -maxdepth 2 -mtime +7 -exec rm -rf {} \;",
             env=self.env,
             workdir='.',
             description=['cleanup', 'old', 'symbols'],
             flunkOnFailure=False,
             mock=self.use_mock,
             target=self.mock_target,
            ))

    def addSourceSteps(self):
        if self.hgHost.startswith('ssh'):
            self.addStep(Mercurial(
             name='hg_ssh_clone',
             mode='update',
             baseURL= '%s/' % self.hgHost,
             defaultBranch=self.repoPath,
             timeout=60*60, # 1 hour
            ))
        elif self.useSharedCheckouts:
            self.addStep(JSONPropertiesDownload(
                name="download_props",
                slavedest="buildprops.json",
                workdir='.'
            ))

            self.addStep(self.makeHgtoolStep(wc='build', workdir='.'))

            self.addStep(SetProperty(
                name = 'set_got_revision',
                command=['hg', 'parent', '--template={node}'],
                extract_fn = short_hash
            ))
        else:
            self.addStep(Mercurial(
             name='hg_update',
             mode='update',
             baseURL='http://%s/' % self.hgHost,
             defaultBranch=self.repoPath,
             timeout=60*60, # 1 hour
            ))

        if self.buildRevision:
            self.addStep(MockCommand(
             name='hg_update',
             command=['hg', 'up', '-C', '-r', self.buildRevision],
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
            ))
            self.addStep(SetProperty(
             name='set_got_revision',
             command=['hg', 'identify', '-i'],
             property='got_revision'
            ))
        #Fix for bug 612319 to correct http://ssh:// changeset links
        if self.hgHost[0:5] == "ssh://":
            changesetLink = '<a href=https://%s/%s/rev' % (self.hgHost[6:],
                                                           self.repoPath)
        else: 
            changesetLink = '<a href=http://%s/%s/rev' % (self.hgHost,
                                                          self.repoPath)
        changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
        self.addStep(OutputStep(
         name='tinderboxprint_changeset',
         data=['TinderboxPrint:', WithProperties(changesetLink)]
        ))
        self.addStep(SetBuildProperty(
            name='set_comments',
            property_name="comments",
            value=lambda build:build.source.changes[-1].comments if len(build.source.changes) > 0 else "",
        ))

    def addConfigSteps(self):
        assert self.configRepoPath is not None
        assert self.configSubDir is not None
        assert self.mozconfig is not None

        configRepo = self.getRepository(self.configRepoPath)
        hg_mozconfig = '%s/raw-file/%s/%s/%s/mozconfig' % (
                configRepo, self.mozconfigBranch, self.configSubDir, self.mozconfig)
        if self.srcMozconfig:
            cmd = ['bash', '-c',
                    '''if [ -f "%(src_mozconfig)s" ]; then
                        echo Using in-tree mozconfig;
                        cp %(src_mozconfig)s .mozconfig;
                    else
                        echo Downloading mozconfig;
                        wget -O .mozconfig %(hg_mozconfig)s;
                    fi'''.replace("\n","") % {'src_mozconfig': self.srcMozconfig, 'hg_mozconfig': hg_mozconfig}]
        else:
            cmd = ['wget', '-O', '.mozconfig', hg_mozconfig]

        self.addStep(RetryingShellCommand(
            name='get_mozconfig',
            command=cmd,
            description=['getting', 'mozconfig'],
            descriptionDone=['got', 'mozconfig'],
            haltOnFailure=True,
        ))
        self.addStep(ShellCommand(
         name='cat_mozconfig',
         command=['cat', '.mozconfig'],
        ))
        if self.tooltool_manifest_src:
          self.addStep(ShellCommand(
            name='run_tooltool',
            command=[
                 WithProperties('%(toolsdir)s/scripts/tooltool/fetch_and_unpack.sh'),
                 self.tooltool_manifest_src,
                 self.tooltool_url_list[0],
                 self.tooltool_script,
                 self.tooltool_bootstrap
            ],
            haltOnFailure=True,
          ))

    def addDoBuildSteps(self):
        workdir=WithProperties('%(basedir)s/build')
        if self.platform.startswith('win'):
            workdir="build"
        # XXX This is a hack but we don't have any intention of adding
        # more branches that are missing MOZ_PGO.  Once these branches
        # are all EOL'd, lets remove this and just put 'build' in the 
        # command argv
        if self.complete_platform == 'win32' and \
           self.branchName in ('mozilla-1.9.1', 'mozilla-1.9.2', 'mozilla-2.0') and \
           self.productName == 'firefox':
            bldtgt = 'profiledbuild'
        else:
            bldtgt = 'build'

        command = self.makeCmd + ['-f', 'client.mk', bldtgt,
                   WithProperties('MOZ_BUILD_DATE=%(buildid:-)s')]

        if self.profiledBuild:
            command.append('MOZ_PGO=1')
        self.addStep(MockCommand(
         name='compile',
         command=command,
         description=['compile'],
         env=self.env,
         haltOnFailure=True,
         timeout=10800,
         # bug 650202 'timeout=7200', bumping to stop the bleeding while we diagnose
         # the root cause of the linker time out.
         workdir=workdir,
         mock=self.use_mock,
         target=self.mock_target,
         mock_workdir_prefix=None,
        ))

    def addBuildInfoSteps(self):
        """Helper function for getting build information into properties.
        Looks for self._gotBuildInfo to make sure we only run this set of steps
        once."""
        if not getattr(self, '_gotBuildInfo', False):
            self.addStep(SetProperty(
                command=['python', 'build%s/config/printconfigsetting.py' % self.mozillaDir,
                'build/%s/dist/bin/application.ini' % self.mozillaObjdir,
                'App', 'BuildID'],
                property='buildid',
                workdir='.',
                description=['getting', 'buildid'],
                descriptionDone=['got', 'buildid'],
            ))
            self.addStep(SetProperty(
                command=['python', 'build%s/config/printconfigsetting.py' % self.mozillaDir,
                'build/%s/dist/bin/application.ini' % self.mozillaObjdir,
                'App', 'SourceStamp'],
                property='sourcestamp',
                workdir='.',
                description=['getting', 'sourcestamp'],
                descriptionDone=['got', 'sourcestamp']
            ))
            self._gotBuildInfo = True

    def addBuildAnalysisSteps(self):
        if self.platform in ('linux', 'linux64'):
            # Analyze the number of ctors
            def get_ctors(rc, stdout, stderr):
                try:
                    output = stdout.split("\t")
                    num_ctors = int(output[0])
                    testresults = [ ('num_ctors', 'num_ctors', num_ctors, str(num_ctors)) ]
                    return dict(num_ctors=num_ctors, testresults=testresults)
                except:
                    return {'testresults': []}

            self.addStep(SetProperty(
                name='get_ctors',
                command=['python', WithProperties('%(toolsdir)s/buildfarm/utils/count_ctors.py'),
                    '%s/dist/bin/libxul.so' % self.mozillaObjdir],
                extract_fn=get_ctors,
                ))

            if self.graphServer:
                self.addBuildInfoSteps()
                self.addStep(JSONPropertiesDownload(slavedest="properties.json"))
                gs_env = self.env.copy()
                gs_env['PYTHONPATH'] = WithProperties('%(toolsdir)s/lib/python')
                self.addStep(GraphServerPost(server=self.graphServer,
                                             selector=self.graphSelector,
                                             branch=self.graphBranch,
                                             resultsname=self.baseName,
                                             env=gs_env,
                                             propertiesFile="properties.json"))
            else:
                self.addStep(OutputStep(
                    name='tinderboxprint_ctors',
                    data=WithProperties('TinderboxPrint: num_ctors: %(num_ctors:-unknown)s'),
                    ))

    def addCheckTestSteps(self):
        env = self.env.copy()
        env['MINIDUMP_STACKWALK'] = getPlatformMinidumpPath(self.platform)
        env['MINIDUMP_SAVE_PATH'] = WithProperties('%(basedir:-)s/minidumps')
        self.addStep(MockMozillaCheck(
                         test_name="check",
                         makeCmd=self.makeCmd,
                         warnOnWarnings=True,
                         workdir=WithProperties('%(basedir)s/build/' + self.objdir),
                         timeout=10 * 60, # 10 minutes.
                         env=env,
                         mock=self.use_mock,
                         target=self.mock_target,
                         mock_workdir_prefix=None,
                     ))

    def addL10nCheckTestSteps(self):
        # We override MOZ_SIGN_CMD here because it's not necessary
        # Disable signing for l10n check steps
        env = self.env
        if 'MOZ_SIGN_CMD' in env:
            env = env.copy()
            del env['MOZ_SIGN_CMD']

        self.addStep(MockCommand(
         name='make l10n check',
         command=self.makeCmd + ['l10n-check'],
         workdir='build/%s' % self.objdir,
         env=env,
         haltOnFailure=False,
         flunkOnFailure=False,
         warnOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))

    def addValgrindCheckSteps(self):
        env = self.env.copy()
        env['MINIDUMP_STACKWALK'] = getPlatformMinidumpPath(self.platform)
        env['MINIDUMP_SAVE_PATH'] = WithProperties('%(basedir:-)s/minidumps')
        self.addStep(unittest_steps.MozillaCheck,
         test_name="check-valgrind",
         warnOnWarnings=True,
         workdir="build/%s/js/src" % self.mozillaObjdir,
         timeout=5*60, # 5 minutes.
         env=env,
        )

    def addCreateUpdateSteps(self):
        self.addStep(ShellCommand(
            name='make_complete_mar',
            command=self.makeCmd + ['-C',
                     '%s/tools/update-packaging' % self.mozillaObjdir],
            env=self.env,
            haltOnFailure=True,
        ))
        self.addFilePropertiesSteps(
            filename='*.complete.mar',
            directory='%s/dist/update' % self.absMozillaObjDir,
            fileType='completeMar',
            haltOnFailure=True,
        )

    def addTestPrettyNamesSteps(self):
        # Disable signing for l10n check steps
        env = self.env
        if 'MOZ_SIGN_CMD' in env:
            env = env.copy()
            del env['MOZ_SIGN_CMD']

        if 'mac' in self.platform:
            # Need to run this target or else the packaging targets will
            # fail.
            self.addStep(ShellCommand(
             name='postflight_all',
             command=self.makeCmd + ['-f', 'client.mk', 'postflight_all'],
             env=env,
             haltOnFailure=False,
             flunkOnFailure=False,
             warnOnFailure=False,
            ))
        pkg_targets = ['package']
        if 'win' in self.platform:
            pkg_targets.append('installer')
        for t in pkg_targets:
            self.addStep(MockCommand(
             name='make %s pretty' % t,
             command=self.makeCmd + [t, 'MOZ_PKG_PRETTYNAMES=1'],
             env=env,
             workdir='build/%s' % self.objdir,
             haltOnFailure=False,
             flunkOnFailure=False,
             warnOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
            ))
        self.addStep(ShellCommand(
             name='make update pretty',
             command=self.makeCmd  +['-C',
                      '%s/tools/update-packaging' % self.mozillaObjdir,
                      'MOZ_PKG_PRETTYNAMES=1'],
             env=env,
             haltOnFailure=False,
             flunkOnFailure=False,
             warnOnFailure=True,
         ))
        if self.l10nCheckTest:
            self.addStep(MockCommand(
                 name='make l10n check pretty',
                command=self.makeCmd + ['l10n-check', 'MOZ_PKG_PRETTYNAMES=1'],
                workdir='build/%s' % self.objdir,
                env=env,
                haltOnFailure=False,
                flunkOnFailure=False,
                warnOnFailure=True,
                mock=self.use_mock,
                target=self.mock_target,
            ))

    def addUploadSteps(self, pkgArgs=None, pkgTestArgs=None):
        pkgArgs = pkgArgs or []
        pkgTestArgs = pkgTestArgs or []

        if self.env:
            pkg_env = self.env.copy()
        else:
            pkg_env = {}
        if self.android_signing:
            pkg_env['JARSIGNER'] = WithProperties('%(toolsdir)s/release/signing/mozpass.py')

        objdir = WithProperties('%(basedir)s/build/' + self.objdir)
        if self.platform.startswith('win'):
            objdir = "build/%s" % self.objdir
        workdir = WithProperties('%(basedir)s/build')
        if self.platform.startswith('win'):
            workdir = "build/"
        if 'rpm' in self.platform_variation:
            pkgArgs.append("MOZ_PKG_FORMAT=RPM")
        if self.packageSDK:
            self.addStep(MockCommand(
             name='make_sdk',
             command=self.makeCmd + ['-f', 'client.mk', 'sdk'],
             env=pkg_env,
             workdir=workdir,
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
             mock_workdir_prefix=None,
            ))
        if self.packageTests:
            self.addStep(MockCommand(
             name='make_pkg_tests',
             command=self.makeCmd + ['package-tests'] + pkgTestArgs,
             env=pkg_env,
             workdir=objdir,
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
             mock_workdir_prefix=None,
            ))
        self.addStep(MockCommand(
            name='make_pkg',
            command=self.makeCmd + ['package'] + pkgArgs,
            env=pkg_env,
            workdir=objdir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
             mock_workdir_prefix=None,
        ))
        # Get package details
        packageFilename = self.getPackageFilename(self.platform,
                                                  self.platform_variation)
        if packageFilename and 'rpm' not in self.platform_variation:
            self.addFilePropertiesSteps(filename=packageFilename,
                                        directory='build/%s/dist' % self.mozillaObjdir,
                                        fileType='package',
                                        haltOnFailure=True)
        # Maemo special cases
        if 'maemo' in self.complete_platform:
            self.addStep(ScratchboxCommand(
                name='make_deb',
                command=self.makeCmd + ['deb'] + pkgArgs,
                env=pkg_env,
                workdir=WithProperties('%(basedir)s/build/' + self.objdir),
                sb=self.use_scratchbox,
                haltOnFailure=True,
            ))
        # Windows special cases
        if self.platform.startswith("win") and \
           'mobile' not in self.complete_platform and \
           self.productName != 'xulrunner':
            self.addStep(MockCommand(
                name='make_installer',
                command=self.makeCmd + ['installer'] + pkgArgs,
                env=pkg_env,
                workdir='build/%s' % self.objdir,
                haltOnFailure=True,
                mock=self.use_mock,
                target=self.mock_target,
            ))
            self.addFilePropertiesSteps(filename='*.installer.exe', 
                                        directory='build/%s/dist/install/sea' % self.mozillaObjdir,
                                        fileType='installer',
                                        haltOnFailure=True)

        if self.productName == 'xulrunner':
            self.addStep(SetProperty(
                command=['python', 'build%s/config/printconfigsetting.py' % self.mozillaDir,
                         'build/%s/dist/bin/platform.ini' % self.mozillaObjdir,
                         'Build', 'BuildID'],
                property='buildid',
                workdir='.',
                name='get_build_id',
            ))
        else:
            self.addStep(SetProperty(
                command=['python', 'build%s/config/printconfigsetting.py' % self.mozillaDir,
                         'build/%s/dist/bin/application.ini' % self.mozillaObjdir,
                         'App', 'BuildID'],
                property='buildid',
                workdir='.',
                name='get_build_id',
            ))
            self.addStep(SetProperty(
                command=['python', 'build%s/config/printconfigsetting.py' % self.mozillaDir,
                         'build/%s/dist/bin/application.ini' % self.mozillaObjdir,
                         'App', 'Version'],
                property='appVersion',
                workdir='.',
                name='get_app_version',
            ))

        if self.multiLocale:
            self.doUpload(postUploadBuildDir='en-US')
            cmd = ['python', 'mozharness/%s' % self.multiLocaleScript,
                   '--config-file', self.multiLocaleConfig]
            if self.multiLocaleMerge:
                cmd.append('--merge-locales')
            cmd.extend(self.mozharnessMultiOptions)
            self.addStep(MockCommand(
                name='mozharness_multilocale',
                command=cmd,
                env=pkg_env,
                workdir='.',
                haltOnFailure=True,
                mock=self.use_mock,
                target=self.mock_target,
            ))
            # We need to set packageFilename to the multi apk
            self.addFilePropertiesSteps(filename=packageFilename,
                                        directory='build/%s/dist' % self.mozillaObjdir,
                                        fileType='package',
                                        haltOnFailure=True)

        if self.createSnippet and 'android' not in self.complete_platform:
            self.addCreateUpdateSteps();

        # Call out to a subclass to do the actual uploading
        self.doUpload(uploadMulti=self.multiLocale)

    def addCreateSnippetsSteps(self, milestone_extra=''):
        if 'android' in self.complete_platform:
            cmd = [
                'python',
                WithProperties('%(toolsdir)s/scripts/android/android_snippet.py'),
                '--download-base-url', self.downloadBaseURL,
                '--download-subdir', self.downloadSubdir,
                '--abi', self.updatePlatform,
                '--aus-base-path', self.ausBaseUploadDir,
                '--aus-host', self.ausHost,
                '--aus-user', self.ausUser,
                '--aus-ssh-key', '~/.ssh/%s' % self.ausSshKey,
                WithProperties(self.objdir + '/dist/%(packageFilename)s')
            ]
            self.addStep(ShellCommand(
                name='create_android_snippet',
                command=cmd,
                haltOnFailure=False,
            ))
        else:
            milestone = self.branchName + milestone_extra
            self.addStep(CreateCompleteUpdateSnippet(
                name='create_complete_snippet',
                objdir=self.absMozillaObjDir,
                milestone=milestone,
                baseurl='%s/nightly' % self.downloadBaseURL,
                hashType=self.hashType,
            ))
            self.addStep(MockCommand(
                name='cat_complete_snippet',
                description=['cat', 'complete', 'snippet'],
                command=['cat', 'complete.update.snippet'],
                workdir='%s/dist/update' % self.absMozillaObjDir,
                mock=self.use_mock,
                target=self.mock_target,
            ))

    def addUploadSnippetsSteps(self):
        self.addStep(RetryingMockCommand(
            name='create_aus_updir',
            command=['bash', '-c',
                     WithProperties('ssh -l %s ' % self.ausUser +
                                    '-i ~/.ssh/%s %s ' % (self.ausSshKey,self.ausHost) +
                                    'mkdir -p %s' % self.ausFullUploadDir)],
            description=['create', 'aus', 'upload', 'dir'],
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(RetryingMockCommand(
            name='upload_complete_snippet',
            command=['scp', '-o', 'User=%s' % self.ausUser,
                     '-o', 'IdentityFile=~/.ssh/%s' % self.ausSshKey,
                     'dist/update/complete.update.snippet',
                     WithProperties('%s:%s/complete.txt' % (self.ausHost,
                                                            self.ausFullUploadDir))],
             workdir=self.absMozillaObjDir,
             description=['upload', 'complete', 'snippet'],
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
        ))
 
    def addUpdateSteps(self):
        self.addCreateSnippetsSteps()
        self.addUploadSnippetsSteps()

    def addBuildSymbolsStep(self):
        objdir = WithProperties('%(basedir)s/build/' + self.objdir)
        if self.platform.startswith('win'):
            objdir = 'build/%s' % self.objdir
        self.addStep(MockCommand(
         name='make_buildsymbols',
         command=self.makeCmd + ['buildsymbols'],
         env=self.env,
         workdir=objdir,
         haltOnFailure=True,
         timeout=60*60*3,
         mock=self.use_mock,
         target=self.mock_target,
         mock_workdir_prefix=None,
        ))

    def addUploadSymbolsStep(self):
        objdir = WithProperties('%(basedir)s/build/' + self.objdir)
        if self.platform.startswith('win'):
            objdir = 'build/%s' % self.objdir
        self.addStep(MockCommand(
         name='make_uploadsymbols',
         command=self.makeCmd + ['uploadsymbols'],
         env=self.env,
         workdir=objdir,
         haltOnFailure=True,
         timeout=2400, # 40 minutes
         mock=self.use_mock,
         target=self.mock_target,
         mock_workdir_prefix=None,
        ))

    def addPostBuildCleanupSteps(self):
        if self.nightly:
            self.addStep(MockCommand(
             name='rm_builddir',
             command=['rm', '-rf', 'build'],
             env=self.env,
             workdir='.',
             timeout=5400, # 1.5 hours
             mock=self.use_mock,
             target=self.mock_target,
            ))

class TryBuildFactory(MercurialBuildFactory):
    def __init__(self,talosMasters=None, unittestMasters=None, packageUrl=None,
                 packageDir=None, unittestBranch=None, tinderboxBuildsDir=None,
                 **kwargs):

        self.packageUrl = packageUrl
        # The directory the packages go into
        self.packageDir = packageDir

        if talosMasters is None:
            self.talosMasters = []
        else:
            assert packageUrl
            self.talosMasters = talosMasters

        self.unittestMasters = unittestMasters or []
        self.unittestBranch = unittestBranch

        if self.unittestMasters:
            assert self.unittestBranch
            assert packageUrl

        self.tinderboxBuildsDir = tinderboxBuildsDir

        MercurialBuildFactory.__init__(self, **kwargs)

    def addSourceSteps(self):
        if self.useSharedCheckouts:
            # We normally rely on the Mercurial step to clobber for us, but
            # since we're managing the checkout ourselves...
            self.addStep(MockCommand(
                name='clobber_build',
                command=['rm', '-rf', 'build'],
                workdir='.',
                timeout=60*60,
                mock=self.use_mock,
                target=self.mock_target,
            ))
            self.addStep(JSONPropertiesDownload(
                name="download_props",
                slavedest="buildprops.json",
                workdir='.'
            ))

            step = self.makeHgtoolStep(
                    clone_by_revision=True,
                    bundles=[],
                    wc='build',
                    workdir='.',
                    locks=[hg_try_lock.access('counting')],
                    )
            self.addStep(step)

        else:
            self.addStep(Mercurial(
            name='hg_update',
            mode='clobber',
            baseURL='http://%s/' % self.hgHost,
            defaultBranch=self.repoPath,
            timeout=60*60, # 1 hour
            locks=[hg_try_lock.access('counting')],
            ))

        if self.buildRevision:
            self.addStep(MockCommand(
             name='hg_update',
             command=['hg', 'up', '-C', '-r', self.buildRevision],
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
            ))
        self.addStep(SetProperty(
         name = 'set_got_revision',
         command=['hg', 'parent', '--template={node}'],
         extract_fn = short_hash
        ))
        changesetLink = '<a href=http://%s/%s/rev' % (self.hgHost,
                                                      self.repoPath)
        changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
        self.addStep(OutputStep(
         name='tinderboxprint_changeset_link',
         data=['TinderboxPrint:', WithProperties(changesetLink)]
        ))
        self.addStep(SetBuildProperty(
            name='set_comments',
            property_name="comments",
            value=lambda build:build.source.changes[-1].comments if len(build.source.changes) > 0 else "",
        ))

    def addLeakTestSteps(self):
        # we want the same thing run a few times here, with different
        # extraArgs
        leakEnv = self.env.copy()
        leakEnv['MINIDUMP_STACKWALK'] = getPlatformMinidumpPath(self.platform)
        leakEnv['MINIDUMP_SAVE_PATH'] = WithProperties('%(basedir:-)s/minidumps')
        for args in [['-register'], ['-CreateProfile', 'default'],
                     ['-P', 'default']]:
            self.addStep(MockAliveTest(
                env=leakEnv,
                workdir='build/%s/_leaktest' % self.mozillaObjdir,
                extraArgs=args,
                warnOnFailure=True,
                haltOnFailure=True,
                mock=self.use_mock,
                target=self.mock_target,
                mock_workdir_prefix=None,
            ))
        baseUrl = 'http://%s/pub/mozilla.org/%s/tinderbox-builds/mozilla-central-%s' % \
            (self.stageServer, self.productName, self.platform)

        self.addLeakTestStepsCommon(baseUrl, leakEnv, False)

    def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
        self.addStep(SetBuildProperty(
             name='set_who',
             property_name='who',
             value=lambda build:str(build.source.changes[0].who) if len(build.source.changes) > 0 else "",
             haltOnFailure=True
        ))

        uploadEnv = self.env.copy()
        uploadEnv.update({
            'UPLOAD_HOST': self.stageServer,
            'UPLOAD_USER': self.stageUsername,
            'UPLOAD_TO_TEMP': '1',
        })

        if self.stageSshKey:
            uploadEnv['UPLOAD_SSH_KEY'] = '~/.ssh/%s' % self.stageSshKey

        # Set up the post upload to the custom try tinderboxBuildsDir
        tinderboxBuildsDir = self.packageDir

        uploadEnv['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
                upload_dir=tinderboxBuildsDir,
                product=self.productName,
                revision=WithProperties('%(got_revision)s'),
                who=WithProperties('%(who)s'),
                builddir=WithProperties('%(branch)s-%(stage_platform)s'),
                buildid=WithProperties('%(buildid)s'),
                to_try=True,
                to_dated=False,
                as_list=False,
                )

        objdir = WithProperties('%(basedir)s/build/' + self.objdir)
        if self.platform.startswith('win'):
            objdir = 'build/%s' % self.objdir
        self.addStep(RetryingMockProperty(
             command=self.makeCmd + ['upload'],
             env=uploadEnv,
             workdir=objdir,
             extract_fn = parse_make_upload,
             haltOnFailure=True,
             description=["upload"],
             timeout=40*60, # 40 minutes
             log_eval_func=lambda c,s: regex_log_evaluator(c, s, upload_errors),
             locks=[upload_lock.access('counting')],
             mock=self.use_mock,
             target=self.mock_target,
             mock_workdir_prefix=None,
        ))

        talosBranch = "%s-%s-talos" % (self.branchName, self.complete_platform)
        sendchange_props = {
                'buildid': WithProperties('%(buildid:-)s'),
                'builduid': WithProperties('%(builduid:-)s'),
                }

        for master, warn, retries in self.talosMasters:
            self.addStep(SendChangeStep(
             name='sendchange_%s' % master,
             warnOnFailure=warn,
             master=master,
             retries=retries,
             branch=talosBranch,
             revision=WithProperties('%(got_revision)s'),
             files=[WithProperties('%(packageUrl)s')],
             user=WithProperties('%(who)s'),
             comments=WithProperties('%(comments:-)s'),
             sendchange_props=sendchange_props,
            ))
        for master, warn, retries in self.unittestMasters:
            self.addStep(SendChangeStep(
             name='sendchange_%s' % master,
             warnOnFailure=warn,
             master=master,
             retries=retries,
             branch=self.unittestBranch,
             revision=WithProperties('%(got_revision)s'),
             files=[WithProperties('%(packageUrl)s'),
                     WithProperties('%(testsUrl)s')],
             user=WithProperties('%(who)s'),
             comments=WithProperties('%(comments:-)s'),
             sendchange_props=sendchange_props,
            ))

class CCMercurialBuildFactory(MercurialBuildFactory):
    def __init__(self, skipBlankRepos=False, mozRepoPath='',
                 inspectorRepoPath='', venkmanRepoPath='',
                 chatzillaRepoPath='', cvsroot='',
                 use_mock=False, mock_target=None,
                 mock_packages=None, mock_copyin_files=None, **kwargs):
        self.skipBlankRepos = skipBlankRepos
        self.mozRepoPath = mozRepoPath
        self.inspectorRepoPath = inspectorRepoPath
        self.venkmanRepoPath = venkmanRepoPath
        self.chatzillaRepoPath = chatzillaRepoPath
        self.cvsroot = cvsroot
        self.use_mock = use_mock
        self.mock_target = mock_target
        self.mock_packages = mock_packages
        self.mock_copyin_files = mock_copyin_files
        MercurialBuildFactory.__init__(self, mozillaDir='mozilla',
            mozconfigBranch='default', use_mock=use_mock, mock_target=mock_target,
            mock_packages=mock_packages, mock_copyin_files=mock_copyin_files, **kwargs)

    def addSourceSteps(self):
        # First set our revisions, if no property by the name, use 'default'
        comm_rev = WithProperties("%(polled_comm_revision:-default)s")
        moz_rev = WithProperties("%(polled_moz_revision:-default)s")

        if self.useSharedCheckouts:
            self.addStep(JSONPropertiesDownload(
                name="download_props",
                slavedest="buildprops.json",
                workdir='.'
            ))

            stepCC = self.makeHgtoolStep(
                wc='build',
                workdir='.',
                rev=comm_rev,
                )
            self.addStep(stepCC)
            
            stepMC = self.makeHgtoolStep(
                name="moz_hg_update",
                wc='build%s' % self.mozillaDir,
                workdir='.',
                rev=moz_rev,
                repo_url=self.getRepository(self.mozRepoPath),
                )
            self.addStep(stepMC)
        else:
            self.addStep(Mercurial(
                name='hg_update',
                mode='update',
                baseURL='http://%s/' % self.hgHost,
                defaultBranch=self.repoPath,
                alwaysUseLatest=True,
                timeout=60*60 # 1 hour
            ))

        if self.buildRevision:
            self.addStep(ShellCommand(
             name='hg_update',
             command=['hg', 'up', '-C', '-r', self.buildRevision],
             haltOnFailure=True,
            ))
        self.addStep(SetProperty(
         name='set_got_revision',
         command=['hg', 'identify', '-i'],
         property='got_revision'
        ))
        #Fix for bug 612319 to correct http://ssh:// changeset links
        if self.hgHost[0:5] == "ssh://":
            changesetLink = '<a href=https://%s/%s/rev' % (self.hgHost[6:],
                                                           self.repoPath)
        else: 
            changesetLink = '<a href=http://%s/%s/rev' % (self.hgHost,
                                                          self.repoPath)
        changesetLink += '/%(got_revision)s title="Built from revision %(got_revision)s">rev:%(got_revision)s</a>'
        self.addStep(OutputStep(
         name='tinderboxprint_changeset',
         data=['TinderboxPrint:', WithProperties(changesetLink)]
        ))
        # build up the checkout command with all options
        co_command = ['python', 'client.py', 'checkout']
        # comm-* is handled by code above, no need to do network churn here
        co_command.append("--skip-comm")
        if (not self.useSharedCheckouts) and self.mozRepoPath:
            co_command.append('--mozilla-repo=%s' % self.getRepository(self.mozRepoPath))
        if self.inspectorRepoPath:
            co_command.append('--inspector-repo=%s' % self.getRepository(self.inspectorRepoPath))
        elif self.skipBlankRepos:
            co_command.append('--skip-inspector')
        if self.venkmanRepoPath:
            co_command.append('--venkman-repo=%s' % self.getRepository(self.venkmanRepoPath))
        elif self.skipBlankRepos:
            co_command.append('--skip-venkman')
        if self.chatzillaRepoPath:
            co_command.append('--chatzilla-repo=%s' % self.getRepository(self.chatzillaRepoPath))
        elif self.skipBlankRepos:
            co_command.append('--skip-chatzilla')
        if self.cvsroot:
            co_command.append('--cvsroot=%s' % self.cvsroot)
        if self.buildRevision:
            co_command.append('--mozilla-rev=%s' % self.buildRevision)
            co_command.append('--inspector-rev=%s' % self.buildRevision)
            co_command.append('--venkman-rev=%s' % self.buildRevision)
            co_command.append('--chatzilla-rev=%s' % self.buildRevision)
        # execute the checkout
        self.addStep(ShellCommand(
         command=co_command,
         name="checkout",
         description=['running', 'client.py', 'checkout'],
         descriptionDone=['client.py', 'checkout'],
         haltOnFailure=True,
         timeout=60*60, # 1 hour
        ))

        self.addStep(SetProperty(
         name='set_hg_revision',
         command=['hg', 'identify', '-i'],
         workdir='build%s' % self.mozillaDir,
         property='hg_revision'
        ))
        #Fix for bug 612319 to correct http://ssh:// changeset links
        if self.hgHost[0:5] == "ssh://":
            changesetLink = '<a href=https://%s/%s/rev' % (self.hgHost[6:],
                                                           self.mozRepoPath)
        else: 
            changesetLink = '<a href=http://%s/%s/rev' % (self.hgHost,
                                                          self.mozRepoPath)
        changesetLink += '/%(hg_revision)s title="Built from Mozilla revision %(hg_revision)s">moz:%(hg_revision)s</a>'
        self.addStep(OutputStep(
         name='tinderboxprint_changeset',
         data=['TinderboxPrint:', WithProperties(changesetLink)]
        ))
        self.addStep(SetBuildProperty(
            name='set_comments',
            property_name="comments",
            value=lambda build:build.source.changes[-1].comments if len(build.source.changes) > 0 else "",
        ))

    def addUploadSteps(self, pkgArgs=None):
        MercurialBuildFactory.addUploadSteps(self, pkgArgs)
        self.addStep(MockCommand(
         name='package_compare',
         command=self.makeCmd + ['package-compare'],
         workdir='build/%s' % self.objdir,
         env=self.env,
         haltOnFailure=False,
         mock=self.use_mock,
         target=self.mock_target,
        ))


def marFilenameToProperty(prop_name=None):
    '''Parse a file listing and return the first mar filename found as
       a named property.
    '''
    def parseMarFilename(rc, stdout, stderr):
        if prop_name is not None:
            for line in filter(None, stdout.split('\n')):
                line = line.strip()
                if re.search(r'\.mar$', line):
                    return {prop_name: line}
        return {}
    return parseMarFilename


class NightlyBuildFactory(MercurialBuildFactory):
    def __init__(self, talosMasters=None, unittestMasters=None,
            unittestBranch=None, tinderboxBuildsDir=None, 
            **kwargs):

        self.talosMasters = talosMasters or []

        self.unittestMasters = unittestMasters or []
        self.unittestBranch = unittestBranch

        if self.unittestMasters:
            assert self.unittestBranch

        self.tinderboxBuildsDir = tinderboxBuildsDir

        MercurialBuildFactory.__init__(self, **kwargs)

    def makePartialTools(self):
        '''The mar and bsdiff tools are created by default when 
           --enable-update-packaging is specified, but some subclasses may 
           need to explicitly build the tools.
        '''
        pass

    def downloadMarTools(self):
        '''The mar and bsdiff tools are created by default when
           --enable-update-packaging is specified. The latest version should
           always be available latest-${branch}/mar-tools/${platform}
           on the ftp server.
        '''
        pass

    def getCompleteMarPatternMatch(self):
        marPattern = getPlatformFtpDir(self.platform)
        if not marPattern:
            return False
        marPattern += '.complete.mar'
        return marPattern

    def previousMarExists(self, step):
        return step.build.getProperties().has_key("previousMarFilename") and len(step.build.getProperty("previousMarFilename")) > 0;

    def addCreatePartialUpdateSteps(self, extraArgs=None):
        '''This function expects that the following build properties are
           already set: buildid, completeMarFilename
        '''
        self.makePartialTools()
        # These tools (mar+mbsdiff) should now be built.
        mar='../dist/host/bin/mar'
        mbsdiff='../dist/host/bin/mbsdiff'
        # Unpack the current complete mar we just made.
        updateEnv = self.env.copy()
        updateEnv['MAR'] = mar
        updateEnv['MBSDIFF'] = mbsdiff
        self.addStep(MockCommand(
            name='rm_unpack_dirs',
            command=['rm', '-rf', 'current', 'previous'],
            env=updateEnv,
            workdir=self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='make_unpack_dirs',
            command=['mkdir', 'current', 'previous'],
            env=updateEnv,
            workdir=self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='unpack_current_mar',
            command=['bash', '-c',
                     WithProperties('%(basedir)s/' +
                                    self.absMozillaSrcDir +
                                    '/tools/update-packaging/unwrap_full_update.pl ' +
                                    '../dist/update/%(completeMarFilename)s')],
            env=updateEnv,
            haltOnFailure=True,
            workdir='%s/current' % self.absMozillaObjDir,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        # The mar file name will be the same from one day to the next,
        # *except* when we do a version bump for a release. To cope with
        # this, we get the name of the previous complete mar directly
        # from staging. Version bumps can also often involve multiple mars
        # living in the latest dir, so we grab the latest one.            
        marPattern = self.getCompleteMarPatternMatch()
        self.addStep(SetProperty(
            name='get_previous_mar_filename',
            description=['get', 'previous', 'mar', 'filename'],
            command=['bash', '-c',
                     WithProperties('ssh -l %s -i ~/.ssh/%s %s ' % (self.stageUsername,
                                                                    self.stageSshKey,
                                                                    self.stageServer) +
                                    'ls -1t %s | grep %s$ | head -n 1' % (self.latestDir,
                                                                        marPattern))
                     ],
            extract_fn=marFilenameToProperty(prop_name='previousMarFilename'),
            flunkOnFailure=False,
            haltOnFailure=False,
            warnOnFailure=True
        ))
        previousMarURL = WithProperties('http://%s' % self.stageServer + \
                          '%s' % self.latestDir + \
                          '/%(previousMarFilename)s')
        self.addStep(RetryingMockCommand(
            name='get_previous_mar',
            description=['get', 'previous', 'mar'],
            doStepIf = self.previousMarExists,
            command=['wget', '-O', 'previous.mar', '--no-check-certificate',
                     previousMarURL],
            workdir='%s/dist/update' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        # Unpack the previous complete mar.
        self.addStep(MockCommand(
            name='unpack_previous_mar',
            description=['unpack', 'previous', 'mar'],
            doStepIf = self.previousMarExists,
            command=['bash', '-c',
                     WithProperties('%(basedir)s/' +
                                    self.absMozillaSrcDir +
                                    '/tools/update-packaging/unwrap_full_update.pl ' +
                                    '../dist/update/previous.mar')],
            env=updateEnv,
            workdir='%s/previous' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        # Extract the build ID from the unpacked previous complete mar.
        self.addStep(FindFile(
            name='find_inipath',
            description=['find', 'inipath'],
            doStepIf = self.previousMarExists,
            filename='application.ini',
            directory='previous',
            filetype='file',
            max_depth=4,
            property_name='previous_inipath',
            workdir=self.absMozillaObjDir,
            haltOnFailure=True,
        ))
        self.addStep(SetProperty(
            name='set_previous_buildid',
            description=['set', 'previous', 'buildid'],
            doStepIf = self.previousMarExists,
            command=['python',
                     '%s/config/printconfigsetting.py' % self.absMozillaSrcDir,
                     WithProperties(self.absMozillaObjDir + '/%(previous_inipath)s'),
                     'App', 'BuildID'],
            property='previous_buildid',
            workdir='.',
            haltOnFailure=True,
        ))
        # Generate the partial patch from the two unpacked complete mars.
        partialMarCommand=self.makeCmd + ['-C',
                           'tools/update-packaging', 'partial-patch',
                           'STAGE_DIR=../../dist/update',
                           'SRC_BUILD=../../previous',
                           WithProperties('SRC_BUILD_ID=%(previous_buildid)s'),
                           'DST_BUILD=../../current',
                           WithProperties('DST_BUILD_ID=%(buildid)s')]
        if extraArgs is not None:
            partialMarCommand.extend(extraArgs)
        self.addStep(MockCommand(
            name='make_partial_mar',
            description=['make', 'partial', 'mar'],
            doStepIf = self.previousMarExists,
            command=partialMarCommand,
            env=updateEnv,
            workdir=self.absMozillaObjDir,
            flunkOnFailure=True,
            haltOnFailure=False,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='rm_previous_mar',
            description=['rm', 'previous', 'mar'],
            doStepIf = self.previousMarExists,
            command=['rm', '-rf', 'previous.mar'],
            env=self.env,
            workdir='%s/dist/update' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        # Update the build properties to pickup information about the partial.
        self.addFilePropertiesSteps(
            filename='*.partial.*.mar',
            doStepIf = self.previousMarExists,
            directory='%s/dist/update' % self.absMozillaObjDir,
            fileType='partialMar',
            haltOnFailure=True,
        )

    def addCreateUpdateSteps(self):
        self.addStep(MockCommand(
            name='rm_existing_mars',
            command=['bash', '-c', 'rm -rvf *.mar'],
            env=self.env,
            workdir='%s/dist/update' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        # Run the parent steps to generate the complete mar.
        MercurialBuildFactory.addCreateUpdateSteps(self)
        if self.createPartial:
            self.addCreatePartialUpdateSteps()

    def addCreateSnippetsSteps(self, milestone_extra=''):
        MercurialBuildFactory.addCreateSnippetsSteps(self, milestone_extra)
        milestone = self.branchName + milestone_extra
        if self.createPartial:
            self.addStep(CreatePartialUpdateSnippet(
                name='create_partial_snippet',
                doStepIf = self.previousMarExists,
                objdir=self.absMozillaObjDir,
                milestone=milestone,
                baseurl='%s/nightly' % self.downloadBaseURL,
                hashType=self.hashType,
            ))
            self.addStep(MockCommand(
                name='cat_partial_snippet',
                description=['cat', 'partial', 'snippet'],
                doStepIf = self.previousMarExists,
                command=['cat', 'partial.update.snippet'],
                workdir='%s/dist/update' % self.absMozillaObjDir,
                mock=self.use_mock,
                target=self.mock_target,
            ))

    def getPreviousBuildUploadDir(self):
        # Uploading the complete snippet occurs regardless of whether we are
        # generating partials on the slave or not, it just goes to a different
        # path for eventual consumption by the central update generation 
        # server.

        # ausFullUploadDir is expected to point to the correct base path on the
        # AUS server for each case:
        #
        # updates generated centrally: /opt/aus2/build/0/...
        # updates generated on slave:  /opt/aus2/incoming/2/...
        if self.createPartial:
            return "%s/%%(previous_buildid)s/en-US" % \
                                         self.ausFullUploadDir
        else:
            return self.ausFullUploadDir
        
    def getCurrentBuildUploadDir(self):
        if self.createPartial:
            return "%s/%%(buildid)s/en-US" % self.ausFullUploadDir
        else:
            return self.ausFullUploadDir

    def addUploadSnippetsSteps(self):
        ausPreviousBuildUploadDir = self.getPreviousBuildUploadDir()
        self.addStep(RetryingShellCommand(
            name='create_aus_previous_updir',
            doStepIf = self.previousMarExists,
            command=['bash', '-c',
                     WithProperties('ssh -l %s ' %  self.ausUser +
                                    '-i ~/.ssh/%s %s ' % (self.ausSshKey,self.ausHost) +
                                    'mkdir -p %s' % ausPreviousBuildUploadDir)],
            description=['create', 'aus', 'previous', 'upload', 'dir'],
            haltOnFailure=True,
            ))
        self.addStep(RetryingMockCommand(
            name='upload_complete_snippet',
            description=['upload', 'complete', 'snippet'],
            doStepIf = self.previousMarExists,
            command=['scp', '-o', 'User=%s' % self.ausUser,
                     '-o', 'IdentityFile=~/.ssh/%s' % self.ausSshKey,
                     'dist/update/complete.update.snippet',
                     WithProperties('%s:%s/complete.txt' % (self.ausHost,
                                                            ausPreviousBuildUploadDir))],
            workdir=self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))

        # We only need to worry about empty snippets (and partials obviously)
        # if we are creating partial patches on the slaves.
        if self.createPartial:
            self.addStep(RetryingMockCommand(
                name='upload_partial_snippet',
                doStepIf = self.previousMarExists,
                command=['scp', '-o', 'User=%s' % self.ausUser,
                         '-o', 'IdentityFile=~/.ssh/%s' % self.ausSshKey,
                         'dist/update/partial.update.snippet',
                         WithProperties('%s:%s/partial.txt' % (self.ausHost,
                                                               ausPreviousBuildUploadDir))],
                workdir=self.absMozillaObjDir,
                description=['upload', 'partial', 'snippet'],
                haltOnFailure=True,
                mock=self.use_mock,
                target=self.mock_target,
            ))
            ausCurrentBuildUploadDir = self.getCurrentBuildUploadDir()
            self.addStep(RetryingShellCommand(
                name='create_aus_current_updir',
                doStepIf = self.previousMarExists,
                command=['bash', '-c',
                         WithProperties('ssh -l %s ' %  self.ausUser +
                                        '-i ~/.ssh/%s %s ' % (self.ausSshKey,self.ausHost) +
                                        'mkdir -p %s' % ausCurrentBuildUploadDir)],
                description=['create', 'aus', 'current', 'upload', 'dir'],
                haltOnFailure=True,
            ))
            # Create remote empty complete/partial snippets for current build.
            # Also touch the remote platform dir to defeat NFS caching on the
            # AUS webheads.
            self.addStep(RetryingShellCommand(
                name='create_empty_snippets',
                doStepIf = self.previousMarExists,
                command=['bash', '-c',
                         WithProperties('ssh -l %s ' %  self.ausUser +
                                        '-i ~/.ssh/%s %s ' % (self.ausSshKey,self.ausHost) +
                                        'touch %s/complete.txt %s/partial.txt %s' % (ausCurrentBuildUploadDir,
                                                                                     ausCurrentBuildUploadDir,
                                                                                     self.ausFullUploadDir))],
                description=['create', 'empty', 'snippets'],
                haltOnFailure=True,
            ))

    def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
        # Because of how the RPM packaging works,
        # we need to tell make upload to look for RPMS
        if 'rpm' in self.complete_platform:
            upload_vars = ["MOZ_PKG_FORMAT=RPM"]
        else:
            upload_vars = []
        uploadEnv = self.env.copy()
        uploadEnv.update({'UPLOAD_HOST': self.stageServer,
                          'UPLOAD_USER': self.stageUsername,
                          'UPLOAD_TO_TEMP': '1'})
        if self.stageSshKey:
            uploadEnv['UPLOAD_SSH_KEY'] = '~/.ssh/%s' % self.stageSshKey

        # Always upload builds to the dated tinderbox builds directories
        if self.tinderboxBuildsDir is None:
            tinderboxBuildsDir = "%s-%s" % (self.branchName, self.stagePlatform)
        else:
            tinderboxBuildsDir = self.tinderboxBuildsDir

        uploadArgs = dict(
                upload_dir=tinderboxBuildsDir,
                product=self.stageProduct,
                buildid=WithProperties("%(buildid)s"),
                revision=WithProperties("%(got_revision)s"),
                as_list=False,
            )
        if self.hgHost.startswith('ssh'):
            uploadArgs['to_shadow'] = True
            uploadArgs['to_tinderbox_dated'] = False
        else:
            uploadArgs['to_shadow'] = False
            uploadArgs['to_tinderbox_dated'] = True

        if self.nightly:
            uploadArgs['to_dated'] = True
            if 'rpm' in self.complete_platform:
                uploadArgs['to_latest'] = False
            else:
                uploadArgs['to_latest'] = True
            if self.post_upload_include_platform:
                # This was added for bug 557260 because of a requirement for
                # mobile builds to upload in a slightly different location
                uploadArgs['branch'] = '%s-%s' % (self.branchName, self.stagePlatform)
            else:
                uploadArgs['branch'] = self.branchName
        if uploadMulti:
            upload_vars.append("AB_CD=multi")
        if postUploadBuildDir:
            uploadArgs['builddir'] = postUploadBuildDir
        uploadEnv['POST_UPLOAD_CMD'] = postUploadCmdPrefix(**uploadArgs)

        if self.productName == 'xulrunner':
            self.addStep(RetryingSetProperty(
             command=self.makeCmd + ['-f', 'client.mk', 'upload'],
             env=uploadEnv,
             workdir='build',
             extract_fn = parse_make_upload,
             haltOnFailure=True,
             description=["upload"],
             timeout=60*60, # 60 minutes
             log_eval_func=lambda c,s: regex_log_evaluator(c, s, upload_errors),
             locks=[upload_lock.access('counting')],
            ))
        else:
            objdir = WithProperties('%(basedir)s/' + self.baseWorkDir + '/' + self.objdir)
            if self.platform.startswith('win'):
                objdir = '%s/%s' % (self.baseWorkDir, self.objdir)
            self.addStep(RetryingMockProperty(
                name='make_upload',
                command=self.makeCmd + ['upload'] + upload_vars,
                env=uploadEnv,
                workdir=objdir,
                extract_fn = parse_make_upload,
                haltOnFailure=True,
                description=['make', 'upload'],
                mock=self.use_mock,
                target=self.mock_target,
                mock_workdir_prefix=None,
                timeout=40*60, # 40 minutes
                log_eval_func=lambda c,s: regex_log_evaluator(c, s, upload_errors),
                locks=[upload_lock.access('counting')],
            ))

        if self.profiledBuild and self.branchName not in ('mozilla-1.9.1', 'mozilla-1.9.2', 'mozilla-2.0'):
            talosBranch = "%s-%s-pgo-talos" % (self.branchName, self.complete_platform)
        else:
            talosBranch = "%s-%s-talos" % (self.branchName, self.complete_platform)

        sendchange_props = {
                'buildid': WithProperties('%(buildid:-)s'),
                'builduid': WithProperties('%(builduid:-)s'),
                }
        if self.nightly:
            sendchange_props['nightly_build'] = True
        # Not sure if this having this property is useful now
        # but it is
        if self.profiledBuild:
            sendchange_props['pgo_build'] = True
        else:
            sendchange_props['pgo_build'] = False

        if not uploadMulti:
            for master, warn, retries in self.talosMasters:
                self.addStep(SendChangeStep(
                 name='sendchange_%s' % master,
                 warnOnFailure=warn,
                 master=master,
                 retries=retries,
                 branch=talosBranch,
                 revision=WithProperties("%(got_revision)s"),
                 files=[WithProperties('%(packageUrl)s')],
                 user="sendchange",
                 comments=WithProperties('%(comments:-)s'),
                 sendchange_props=sendchange_props,
                 env=self.env,
                ))

            files = [WithProperties('%(packageUrl)s')]
            if '1.9.1' not in self.branchName:
                files.append(WithProperties('%(testsUrl)s'))

            for master, warn, retries in self.unittestMasters:
                self.addStep(SendChangeStep(
                 name='sendchange_%s' % master,
                 warnOnFailure=warn,
                 master=master,
                 retries=retries,
                 branch=self.unittestBranch,
                 revision=WithProperties("%(got_revision)s"),
                 files=files,
                 user="sendchange-unittest",
                 comments=WithProperties('%(comments:-)s'),
                 sendchange_props=sendchange_props,
                 env=self.env,
                ))


class CCNightlyBuildFactory(CCMercurialBuildFactory, NightlyBuildFactory):
    def __init__(self, skipBlankRepos=False, mozRepoPath='',
                 inspectorRepoPath='', venkmanRepoPath='',
                 chatzillaRepoPath='', cvsroot='',
                 use_mock=False, mock_target=None, mock_packages=None,
                 mock_copyin_files=None, **kwargs):
        self.skipBlankRepos = skipBlankRepos
        self.mozRepoPath = mozRepoPath
        self.inspectorRepoPath = inspectorRepoPath
        self.venkmanRepoPath = venkmanRepoPath
        self.chatzillaRepoPath = chatzillaRepoPath
        self.cvsroot = cvsroot
        self.use_mock = use_mock
        self.mock_target = mock_target
        self.mock_packages = mock_packages
        self.mock_copyin_files = mock_copyin_files
        NightlyBuildFactory.__init__(self, mozillaDir='mozilla',
            mozconfigBranch='default', use_mock=use_mock, mock_target=mock_target,
            mock_packages=mock_packages, mock_copyin_files=mock_copyin_files, **kwargs)

    # MercurialBuildFactory defines those, and our inheritance chain makes us
    # look there before NightlyBuildFactory, so we need to define them here and
    # call the actually wanted implementation.
    def addCreateUpdateSteps(self):
        NightlyBuildFactory.addCreateUpdateSteps(self)

    def addCreateSnippetsSteps(self, milestone_extra=''):
        NightlyBuildFactory.addCreateSnippetsSteps(self, milestone_extra)

    def addUploadSnippetsSteps(self):
        NightlyBuildFactory.addUploadSnippetsSteps(self)


class ReleaseBuildFactory(MercurialBuildFactory):
    def __init__(self, env, version, buildNumber, brandName=None,
            unittestMasters=None, unittestBranch=None, talosMasters=None,
            usePrettyNames=True, enableUpdatePackaging=True, oldVersion=None,
            oldBuildNumber=None, appVersion=None, use_mock=False,
            mock_target=None, mock_packages=None, mock_copyin_files=None, **kwargs):
        self.version = version
        self.buildNumber = buildNumber

        self.talosMasters = talosMasters or []
        self.unittestMasters = unittestMasters or []
        self.unittestBranch = unittestBranch
        self.enableUpdatePackaging = enableUpdatePackaging
        if self.unittestMasters:
            assert self.unittestBranch
        self.oldVersion = oldVersion
        self.oldBuildNumber = oldBuildNumber
        self.use_mock = use_mock
        self.mock_target = mock_target
        self.mock_copyin_files = mock_copyin_files
        self.mock_packages = mock_packages
        if brandName:
            self.brandName = brandName
        else:
            self.brandName = kwargs['productName'].capitalize()
        self.UPLOAD_EXTRA_FILES = []
        # Copy the environment to avoid screwing up other consumers of
        # MercurialBuildFactory
        env = env.copy()
        # Make sure MOZ_PKG_PRETTYNAMES is on and override MOZ_PKG_VERSION
        # The latter is only strictly necessary for RCs.
        if usePrettyNames:
            env['MOZ_PKG_PRETTYNAMES'] = '1'
        if appVersion is None or self.version != appVersion:
            env['MOZ_PKG_VERSION'] = version
        MercurialBuildFactory.__init__(self, env=env, **kwargs)

    def addFilePropertiesSteps(self, filename=None, directory=None,
                               fileType=None, maxDepth=1, haltOnFailure=False):
        # We don't need to do this for release builds.
        pass

    def addCreatePartialUpdateSteps(self):
        updateEnv = self.env.copy()
        # need to use absolute paths since mac may have difeerent layout
        # objdir/arch/dist vs objdir/dist
        updateEnv['MAR'] = WithProperties(
            '%(basedir)s' + '/%s/dist/host/bin/mar' % self.absMozillaObjDir)
        updateEnv['MBSDIFF'] = WithProperties(
            '%(basedir)s' + '/%s/dist/host/bin/mbsdiff' % self.absMozillaObjDir)
        current_mar_name = '%s-%s.complete.mar' % (self.productName,
                                                   self.version)
        previous_mar_name = '%s-%s.complete.mar' % (self.productName,
                                                    self.oldVersion)
        partial_mar_name = '%s-%s-%s.partial.mar' % \
            (self.productName, self.oldVersion, self.version)
        oldCandidatesDir = makeCandidatesDir(
            self.productName, self.oldVersion, self.oldBuildNumber,
            protocol='http', server=self.stageServer)
        previousMarURL = '%s/update/%s/en-US/%s' % \
            (oldCandidatesDir, getPlatformFtpDir(self.platform),
             previous_mar_name)
        mar_unpack_cmd = WithProperties(
            '%(basedir)s' + '/%s/tools/update-packaging/unwrap_full_update.pl' % self.absMozillaSrcDir)
        partial_mar_cmd = WithProperties(
            '%(basedir)s' + '/%s/tools/update-packaging/make_incremental_update.sh' % self.absMozillaSrcDir)
        update_dir = 'update/%s/en-US' % getPlatformFtpDir(self.platform)

        self.addStep(MockCommand(
            name='rm_unpack_dirs',
            command=['rm', '-rf', 'current', 'previous'],
            env=updateEnv,
            workdir=self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='make_unpack_dirs',
            command=['mkdir', 'current', 'previous'],
            env=updateEnv,
            workdir=self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(RetryingMockCommand(
            name='get_previous_mar',
            description=['get', 'previous', 'mar'],
            command=['wget', '-O', 'previous.mar', '--no-check-certificate',
                     previousMarURL],
            workdir='%s/dist' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='unpack_previous_mar',
            description=['unpack', 'previous', 'mar'],
            command=['perl', mar_unpack_cmd, '../dist/previous.mar'],
            env=updateEnv,
            workdir='%s/previous' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='unpack_current_mar',
            command=['perl', mar_unpack_cmd,
                     '../dist/%s/%s' % (update_dir, current_mar_name)],
            env=updateEnv,
            haltOnFailure=True,
            workdir='%s/current' % self.absMozillaObjDir,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='make_partial_mar',
            description=self.makeCmd + ['partial', 'mar'],
            command=['bash', partial_mar_cmd,
                     '%s/%s' % (update_dir, partial_mar_name),
                     '../previous', '../current'],
            env=updateEnv,
            workdir='%s/dist' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        # the previous script exits 0 always, need to check if mar exists
        self.addStep(MockCommand(
                    name='check_partial_mar',
                    description=['check', 'partial', 'mar'],
                    command=['ls', '-l',
                             '%s/%s' % (update_dir, partial_mar_name)],
                    workdir='%s/dist' % self.absMozillaObjDir,
                    haltOnFailure=True,
                    mock=self.use_mock,
                    target=self.mock_target,
        ))
        self.UPLOAD_EXTRA_FILES.append('%s/%s' % (update_dir, partial_mar_name))
        if self.enableSigning and self.signingServers:
            partial_mar_path = '"%s/dist/%s/%s"' % \
                (self.absMozillaObjDir, update_dir, partial_mar_name)
            cmd = '%s -f gpg -f mar "%s"' % (self.signing_command,
                                             partial_mar_path)
            self.addStep(MockCommand(
                name='sign_partial_mar',
                description=['sign', 'partial', 'mar'],
                command=['bash', '-c', WithProperties(cmd)],
                env=updateEnv,
                workdir='.',
                haltOnFailure=True,
                mock=self.use_mock,
                target=self.mock_target,
            ))
            self.UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (update_dir,
                                                          partial_mar_name))

    def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
        info_txt = '%s_info.txt' % self.complete_platform
        self.UPLOAD_EXTRA_FILES.append(info_txt)
        # Make sure the complete MAR has been generated
        if self.enableUpdatePackaging:
            self.addStep(ShellCommand(
                name='make_update_pkg',
                command=self.makeCmd + ['-C',
                         '%s/tools/update-packaging' % self.mozillaObjdir],
                env=self.env,
                haltOnFailure=True,
            ))
            if self.createPartial:
                self.addCreatePartialUpdateSteps()
        self.addStep(MockCommand(
         name='echo_buildID',
         command=['bash', '-c',
                  WithProperties('echo buildID=%(buildid)s > ' + info_txt)],
         workdir='build/%s/dist' % self.mozillaObjdir,
         mock=self.use_mock,
         target=self.mock_target,
        ))

        uploadEnv = self.env.copy()
        uploadEnv.update({'UPLOAD_HOST': self.stageServer,
                          'UPLOAD_USER': self.stageUsername,
                          'UPLOAD_TO_TEMP': '1',
                          'UPLOAD_EXTRA_FILES': ' '.join(self.UPLOAD_EXTRA_FILES)})
        if self.stageSshKey:
            uploadEnv['UPLOAD_SSH_KEY'] = '~/.ssh/%s' % self.stageSshKey

        uploadArgs = dict(
            upload_dir="%s-%s" % (self.branchName, self.platform),
            product=self.productName,
            version=self.version,
            buildNumber=str(self.buildNumber),
            as_list=False)
        if self.enableSigning and self.signingServers:
            uploadArgs['signed'] = True
        upload_vars = []

        if self.productName == 'fennec':
            builddir = '%s/en-US' % self.stagePlatform
            if uploadMulti:
                builddir = '%s/multi' % self.stagePlatform
                upload_vars = ['AB_CD=multi']
            if postUploadBuildDir:
                builddir = '%s/%s' % (self.stagePlatform, postUploadBuildDir)
            uploadArgs['builddir'] = builddir
            uploadArgs['to_mobile_candidates'] = True
            uploadArgs['nightly_dir'] = 'candidates'
            uploadArgs['product'] = 'mobile'
        else:
            uploadArgs['to_candidates'] = True

        uploadEnv['POST_UPLOAD_CMD'] = postUploadCmdPrefix(**uploadArgs)

        objdir = WithProperties('%(basedir)s/build/' + self.objdir)
        if self.platform.startswith('win'):
            objdir = 'build/%s' % self.objdir
        self.addStep(RetryingScratchboxProperty(
         name='make_upload',
         command=self.makeCmd + ['upload'] + upload_vars,
         env=uploadEnv,
         workdir=objdir,
         extract_fn=parse_make_upload,
         haltOnFailure=True,
         description=['upload'],
         timeout=60*60, # 60 minutes
         log_eval_func=lambda c,s: regex_log_evaluator(c, s, upload_errors),
         sb=self.use_scratchbox,
        ))

        if self.productName == 'fennec' and not uploadMulti:
            cmd = ['scp']
            if self.stageSshKey:
                cmd.append('-oIdentityFile=~/.ssh/%s' % self.stageSshKey)
            cmd.append(info_txt)
            candidates_dir = makeCandidatesDir(self.productName, self.version,
                                               self.buildNumber)
            cmd.append('%s@%s:%s' % (self.stageUsername, self.stageServer,
                                     candidates_dir))
            self.addStep(RetryingShellCommand(
                name='upload_buildID',
                command=cmd,
                workdir='build/%s/dist' % self.mozillaObjdir
            ))

        # Send to the "release" branch on talos, it will do
        # super-duper-extra testing
        talosBranch = "release-%s-%s-talos" % (self.branchName, self.complete_platform)
        sendchange_props = {
                'buildid': WithProperties('%(buildid:-)s'),
                'builduid': WithProperties('%(builduid:-)s'),
                }
        for master, warn, retries in self.talosMasters:
            self.addStep(SendChangeStep(
             name='sendchange_%s' % master,
             warnOnFailure=warn,
             master=master,
             retries=retries,
             branch=talosBranch,
             revision=WithProperties("%(got_revision)s"),
             files=[WithProperties('%(packageUrl)s')],
             user="sendchange",
             comments=WithProperties('%(comments:-)s'),
             sendchange_props=sendchange_props,
            ))

        for master, warn, retries in self.unittestMasters:
            self.addStep(SendChangeStep(
             name='sendchange_%s' % master,
             warnOnFailure=warn,
             master=master,
             retries=retries,
             branch=self.unittestBranch,
             revision=WithProperties("%(got_revision)s"),
             files=[WithProperties('%(packageUrl)s'),
                    WithProperties('%(testsUrl)s')],
             user="sendchange-unittest",
             comments=WithProperties('%(comments:-)s'),
             sendchange_props=sendchange_props,
            ))

class XulrunnerReleaseBuildFactory(ReleaseBuildFactory):
    def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
        uploadEnv = self.env.copy()
        uploadEnv.update({'UPLOAD_HOST': self.stageServer,
                          'UPLOAD_USER': self.stageUsername,
                          'UPLOAD_TO_TEMP': '1'})
        if self.stageSshKey:
            uploadEnv['UPLOAD_SSH_KEY'] = '~/.ssh/%s' % self.stageSshKey

        uploadEnv['POST_UPLOAD_CMD'] = 'post_upload.py ' + \
                                       '-p %s ' % self.productName + \
                                       '-v %s ' % self.version + \
                                       '-n %s ' % self.buildNumber + \
                                       '--release-to-candidates-dir'
        def get_url(rc, stdout, stderr):
            for m in re.findall("^(http://.*?\.(?:tar\.bz2|dmg|zip))", "\n".join([stdout, stderr]), re.M):
                if m.endswith("crashreporter-symbols.zip"):
                    continue
                if m.endswith("tests.tar.bz2"):
                    continue
                return {'packageUrl': m}
            return {'packageUrl': ''}

        self.addStep(RetryingSetProperty(
         command=self.makeCmd + ['-f', 'client.mk', 'upload'],
         env=uploadEnv,
         workdir='build',
         extract_fn = get_url,
         haltOnFailure=True,
         description=['upload'],
         log_eval_func=lambda c,s: regex_log_evaluator(c, s, upload_errors),
        ))

class CCReleaseBuildFactory(CCMercurialBuildFactory, ReleaseBuildFactory):
    def __init__(self, mozRepoPath='', inspectorRepoPath='',
                 venkmanRepoPath='', chatzillaRepoPath='', cvsroot='',
                 use_mock=False, mock_target=None, mock_packages=None,
                 mock_copyin_files=None,
                 **kwargs):
        self.skipBlankRepos = True
        self.mozRepoPath = mozRepoPath
        self.inspectorRepoPath = inspectorRepoPath
        self.venkmanRepoPath = venkmanRepoPath
        self.chatzillaRepoPath = chatzillaRepoPath
        self.cvsroot = cvsroot
        self.use_mock = use_mock
        self.mock_target = mock_target
        self.mock_packages = mock_packages
        self.mock_copyin_files = mock_copyin_files
        ReleaseBuildFactory.__init__(self, mozillaDir='mozilla',
            mozconfigBranch='default', use_mock=False, mock_target=None,
            mock_packages=None, mock_copyin_files=None, **kwargs)

    def addFilePropertiesSteps(self, filename=None, directory=None,
                               fileType=None, maxDepth=1, haltOnFailure=False):
        # We don't need to do this for release builds.
        pass


def identToProperties(default_prop=None):
    '''Create a method that is used in a SetProperty step to map the
    output of make ident to build properties.

    To be backwards compat, this allows for a property name to be specified
    to be used for a single hg revision.
    '''
    def list2dict(rv, stdout, stderr):
        props = {}
        stdout = stdout.strip()
        if default_prop is not None and re.match(r'[0-9a-f]{12}\+?', stdout):
            # a single hg version
            props[default_prop] = stdout
        else:
            for l in filter(None, stdout.split('\n')):
                e = filter(None, l.split())
                props[e[0]] = e[1]
        return props
    return list2dict


class BaseRepackFactory(MozillaBuildFactory):
    # Override ignore_dirs so that we don't delete l10n nightly builds
    # before running a l10n nightly build
    ignore_dirs = MozillaBuildFactory.ignore_dirs + [reallyShort('*-nightly')]

    extraConfigureArgs = []

    def __init__(self, project, appName, l10nRepoPath,
                 compareLocalesRepoPath, compareLocalesTag, stageServer,
                 stageUsername, stageSshKey=None, objdir='', platform='',
                 mozconfig=None,
                 tree="notset", mozillaDir=None, l10nTag='default',
                 mergeLocales=True,
                 testPrettyNames=False,
                 tooltool_manifest_src=None,
                 tooltool_bootstrap="setup.sh",
                 tooltool_url_list=None,
                 tooltool_script='/tools/tooltool.py',
                 use_mock=None,
                 mock_target=None,
                 **kwargs):
        MozillaBuildFactory.__init__(self, **kwargs)

        self.platform = platform
        self.project = project
        self.productName = project
        self.appName = appName
        self.l10nRepoPath = l10nRepoPath
        self.l10nTag = l10nTag
        self.compareLocalesRepoPath = compareLocalesRepoPath
        self.compareLocalesTag = compareLocalesTag
        self.mergeLocales = mergeLocales
        self.stageServer = stageServer
        self.stageUsername = stageUsername
        self.stageSshKey = stageSshKey
        self.tree = tree
        self.mozconfig = mozconfig
        self.testPrettyNames = testPrettyNames
        self.tooltool_manifest_src = tooltool_manifest_src
        self.tooltool_url_list = tooltool_url_list or []
        self.tooltool_script = tooltool_script
        self.tooltool_bootstrap = tooltool_bootstrap
        self.use_mock = use_mock
        self.mock_target = mock_target

        assert len(self.tooltool_url_list) <= 1, "multiple urls not currently supported by tooltool"

        self.addStep(SetBuildProperty(
         property_name='tree',
         value=self.tree,
         haltOnFailure=True
        ))

        self.origSrcDir = self.branchName

        if self.enable_pymake:
            self.makeCmd = ['python2.7', WithProperties("%(basedir)s/build/" + "%s/build/pymake/make.py" % self.origSrcDir)]
        else:
            self.makeCmd = ['make']

        # Mozilla subdir
        if mozillaDir:
            self.mozillaDir = '/%s' % mozillaDir
            self.mozillaSrcDir = '%s/%s' % (self.origSrcDir, mozillaDir)
        else:
            self.mozillaDir = ''
            self.mozillaSrcDir = self.origSrcDir

        # self.mozillaObjdir is used in SeaMonkey's and Thunderbird's case
        self.objdir = objdir or self.origSrcDir
        self.mozillaObjdir = '%s%s' % (self.objdir, self.mozillaDir)

        self.absSrcDir = "%s/%s" % (self.baseWorkDir, self.origSrcDir)
        self.absObjDir = "%s/%s" % (self.absSrcDir, self.objdir)

        # These following variables are useful for sharing build steps (e.g.
        # update generation) from classes that use object dirs (e.g. nightly
        # repacks).
        # 
        # We also concatenate the baseWorkDir at the outset to avoid having to
        # do that everywhere.
        self.absMozillaSrcDir = "%s/%s" % (self.baseWorkDir, self.mozillaSrcDir)
        self.absMozillaObjDir = '%s/%s/%s' % (self.baseWorkDir, self.origSrcDir, self.mozillaObjdir)

        self.latestDir = '/pub/mozilla.org/%s' % self.productName + \
                         '/nightly/latest-%s-l10n' % self.branchName
        
        if objdir != '':
            # L10NBASEDIR is relative to MOZ_OBJDIR
            self.env.update({'MOZ_OBJDIR': objdir,
                             'L10NBASEDIR':  '../../l10n'})

        if platform == 'macosx64':
            # use "mac" instead of "mac64" for macosx64
            self.env.update({'MOZ_PKG_PLATFORM': 'mac'})

        self.uploadEnv = self.env.copy() # pick up any env variables in our subclass
        self.uploadEnv.update({
            'AB_CD': WithProperties('%(locale)s'),
            'UPLOAD_HOST': stageServer,
            'UPLOAD_USER': stageUsername,
            'UPLOAD_TO_TEMP': '1',
            'POST_UPLOAD_CMD': self.postUploadCmd # defined in subclasses
        })
        if stageSshKey:
            self.uploadEnv['UPLOAD_SSH_KEY'] = '~/.ssh/%s' % stageSshKey

        self.preClean()

        # Need to override toolsdir as set by MozillaBuildFactory because
        # we need Windows-style paths.
        if self.platform.startswith('win'):
            self.addStep(SetProperty(
                command=['bash', '-c', 'pwd -W'],
                property='toolsdir',
                workdir='tools'
            ))
            self.addStep(SetProperty(
                command=['bash', '-c', 'pwd -W'],
                property='basedir',
                workdir='.'
            ))
        self.addStep(ShellCommand(
         name='mkdir_l10n',
         command=['sh', '-c', 'mkdir -p l10n'],
         descriptionDone='mkdir l10n',
         workdir=self.baseWorkDir,
         flunkOnFailure=False
        ))

        # call out to overridable functions
        self.getSources()
        self.updateSources()
        self.getMozconfig()
        self.configure()
        self.tinderboxPrintBuildInfo()
        self.downloadBuilds()
        self.updateEnUS()
        self.tinderboxPrintRevisions()
        self.compareLocalesSetup()
        self.compareLocales()
        if self.signingServers and self.enableSigning:
            self.addGetTokenSteps()
        self.doRepack()
        self.doUpload()
        if self.testPrettyNames:
            self.doTestPrettyNames()

    def processCommand(self, **kwargs):
        '''This function is overriden by MaemoNightlyRepackFactory to
        adjust the command and workdir approprietaly for scratchbox
        '''
        return kwargs

    def getMozconfig(self):
        if not self.mozconfig:
            return

        cmd = ['bash', '-c',
               '''if [ -f "%(mozconfig)s" ]; then
                   echo Using in-tree mozconfig;
                   cp %(mozconfig)s .mozconfig;
                else
                   echo Could not find in-tree mozconfig;
                   exit 1;
                fi'''.replace("\n","") % {'mozconfig': self.mozconfig}]

        self.addStep(RetryingMockCommand(
            name='get_mozconfig',
            command=cmd,
            description=['getting', 'mozconfig'],
            descriptionDone=['got', 'mozconfig'],
            workdir='build/'+self.origSrcDir,
            haltOnFailure=False,
            flunkOnFailure=False,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='cat_mozconfig',
            command=['cat', '.mozconfig'],
            workdir='build/'+self.origSrcDir,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        if self.tooltool_manifest_src:
          self.addStep(MockCommand(
            name='run_tooltool',
            command=[
                 WithProperties('%(toolsdir)s/scripts/tooltool/fetch_and_unpack.sh'),
                 self.tooltool_manifest_src,
                 self.tooltool_url_list[0],
                 self.tooltool_script,
                 self.tooltool_bootstrap
            ],
            haltOnFailure=True,
            workdir='%s/%s' % (self.baseWorkDir, self.origSrcDir),
            mock=self.use_mock,
            target=self.mock_target,
          ))


    def configure(self):
        if self.mozillaDir:
            self.addStep(MockCommand( **self.processCommand(
                name='link_tools',
                env=self.env,
                command=['sh', '-c', 'if [ ! -e tools ]; then ln -s ../tools; fi'],
                workdir='build',
                description=['link tools'],
                mock=self.use_mock,
                target=self.mock_target,
            )))
        self.addStep(MockCommand( **self.processCommand(
            name='configure',
            command=self.makeCmd + [ '-f', 'client.mk', 'configure'],
            workdir=self.absSrcDir,
            description='configure',
            descriptionDone='configure done',
            env=self.env,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        )))
        self.addStep(MockCommand( **self.processCommand(
         name='make_config',
         command=self.makeCmd,
         workdir='%s/config' % self.absMozillaObjDir,
         description=['make config'],
         env=self.env,
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        )))

    def tinderboxPrint(self, propName, propValue):
        self.addStep(OutputStep(
                     name='tinderboxprint_%s' % propName,
                     data=['TinderboxPrint:',
                           '%s:' % propName,
                           propValue]
        ))

    def tinderboxPrintBuildInfo(self):
        '''Display some build properties for scraping in Tinderbox.
        '''
        self.tinderboxPrint('locale',WithProperties('%(locale)s'))
        self.tinderboxPrint('tree',self.tree)
        self.tinderboxPrint('buildnumber',WithProperties('%(buildnumber)s'))

    def doUpload(self, postUploadBuildDir=None, uploadMulti=False):
        self.addStep(RetryingMockCommand(
         name='make_upload',
         command=self.makeCmd + ['upload', WithProperties('AB_CD=%(locale)s')],
         env=self.uploadEnv,
         workdir='%s/%s/locales' % (self.absObjDir, self.appName),
         haltOnFailure=True,
         flunkOnFailure=True,
         log_eval_func=lambda c,s: regex_log_evaluator(c, s, upload_errors),
         locks=[upload_lock.access('counting')],
         mock=self.use_mock,
         target=self.mock_target,
         mock_workdir_prefix=None,
        ))

    def getSources(self):
        step = self.makeHgtoolStep(
                name='get_enUS_src',
                wc=self.origSrcDir,
                rev=WithProperties("%(en_revision)s"),
                locks=[hg_l10n_lock.access('counting')],
                workdir=self.baseWorkDir,
                use_properties=False,
                )
        self.addStep(step)

        step = self.makeHgtoolStep(
                name='get_locale_src',
                rev=WithProperties("%(l10n_revision)s"),
                repo_url=WithProperties("http://" + self.hgHost + "/" + \
                                 self.l10nRepoPath + "/%(locale)s"),
                workdir='%s/l10n' % self.baseWorkDir,
                locks=[hg_l10n_lock.access('counting')],
                use_properties=False,
                mirrors=[],
                )
        self.addStep(step)

    def updateEnUS(self):
        '''Update the en-US source files to the revision used by
        the repackaged build.

        This is implemented in the subclasses.
        '''
        pass

    def tinderboxPrintRevisions(self):
        '''Display the various revisions used in building for
        scraping in Tinderbox.
        This is implemented in the subclasses.
        '''  
        pass

    def compareLocalesSetup(self):
        compareLocalesRepo = self.getRepository(self.compareLocalesRepoPath)
        self.addStep(MockCommand(
         name='rm_compare_locales',
         command=['rm', '-rf', 'compare-locales'],
         description=['remove', 'compare-locales'],
         workdir=self.baseWorkDir,
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MercurialCloneCommand(
         name='clone_compare_locales',
         command=['hg', 'clone', compareLocalesRepo, 'compare-locales'],
         description=['checkout', 'compare-locales'],
         workdir=self.baseWorkDir,
         haltOnFailure=True
        ))
        self.addStep(MockCommand(
         name='update_compare_locales',
         command=['hg', 'up', '-C', '-r', self.compareLocalesTag],
         description='update compare-locales',
         workdir='%s/compare-locales' % self.baseWorkDir,
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))

    def compareLocales(self):
        if self.mergeLocales:
            mergeLocaleOptions = ['-m',
                                  WithProperties('%(basedir)s/' + \
                                                 "%s/merged" % self.baseWorkDir)]
                ###'%s/%s/locales/merged' % (self.absObjDir, self.appName)]
            flunkOnFailure = False
            haltOnFailure = False
            warnOnFailure = True
        else:
            mergeLocaleOptions = []
            flunkOnFailure = True
            haltOnFailure = True
            warnOnFailure = False
        self.addStep(MockCommand(
         name='rm_merged',
         command=['rm', '-rf', 'merged'],
         description=['remove', 'merged'],
         workdir=self.baseWorkDir,
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MockCommand(
         name='run_compare_locales',
#         command=['python',
#                  'build/compare-locales/scripts/compare-locales'] +
#                  mergeLocaleOptions +
#                  ["%s/%s/locales/l10n.ini" % (self.absSrcDir, self.appName),
#                  "build/l10n",
#                  WithProperties('%(locale)s')],
         command=['python',
                  '../../../compare-locales/scripts/compare-locales'] +
                  mergeLocaleOptions +
                  ["l10n.ini",
                   "../../../l10n",
                   WithProperties('%(locale)s')],
         description='comparing locale',
         env={'PYTHONPATH': ['../../../compare-locales/lib']},
#         env={'PYTHONPATH': ['build/compare-locales/lib']},
         flunkOnFailure=flunkOnFailure,
         warnOnFailure=warnOnFailure,
         haltOnFailure=haltOnFailure,
         workdir="%s/%s/locales" % (self.absSrcDir,
                                    self.appName),
         mock=self.use_mock,
         target=self.mock_target,
#         workdir=".",
        ))

    def doRepack(self):
        '''Perform the repackaging.

        This is implemented in the subclasses.
        '''
        pass

    def preClean(self):
        self.addStep(MockCommand(
         name='rm_dist_upload',
         command=['sh', '-c',
                  'if [ -d '+self.mozillaObjdir+'/dist/upload ]; then ' +
                  'rm -rf '+self.mozillaObjdir+'/dist/upload; ' +
                  'fi'],
         description="rm dist/upload",
         workdir=self.baseWorkDir,
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))

        self.addStep(MockCommand(
         name='rm_dist_update',
         command=['sh', '-c',
                  'if [ -d '+self.mozillaObjdir+'/dist/update ]; then ' +
                  'rm -rf '+self.mozillaObjdir+'/dist/update; ' +
                  'fi'],
         description="rm dist/update",
         workdir=self.baseWorkDir,
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))

    def doTestPrettyNames(self):
        # Need to re-download this file because it gets removed earlier
        self.addStep(MockCommand(
         name='wget_enUS',
         command=self.makeCmd + ['wget-en-US'],
         description='wget en-US',
         env=self.env,
         haltOnFailure=True,
         workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MockCommand(
         name='make_unpack',
         command=self.makeCmd + ['unpack'],
         description='unpack en-US',
         haltOnFailure=True,
         env=self.env,
         workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
         mock=self.use_mock,
         target=self.mock_target,
        ))
        # We need to override ZIP_IN because it defaults to $(PACKAGE), which
        # will be the pretty name version here.
        self.addStep(SetProperty(
         command=self.makeCmd + ['--no-print-directory', 'echo-variable-ZIP_IN'],
         property='zip_in',
         env=self.env,
         workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
         haltOnFailure=True,
        ))
        prettyEnv = self.env.copy()
        prettyEnv['MOZ_PKG_PRETTYNAMES'] = '1'
        prettyEnv['ZIP_IN'] = WithProperties('%(zip_in)s')
        if self.platform.startswith('win'):
            self.addStep(SetProperty(
             command=self.makeCmd + ['--no-print-directory', 'echo-variable-WIN32_INSTALLER_IN'],
             property='win32_installer_in',
             env=self.env,
             workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
             haltOnFailure=True,
            ))
            prettyEnv['WIN32_INSTALLER_IN'] = WithProperties('%(win32_installer_in)s')
        self.addStep(MockCommand(
         name='repack_installers_pretty',
         description=['repack', 'installers', 'pretty'],
         command=['sh', '-c',
                  WithProperties('make installers-%(locale)s LOCALE_MERGEDIR=$PWD/merged')],
         env=prettyEnv,
         haltOnFailure=False,
         flunkOnFailure=False,
         warnOnFailure=True,
         workdir='%s/%s/locales' % (self.absObjDir, self.appName),
         mock=self.use_mock,
         target=self.mock_target,
        ))

class CCBaseRepackFactory(BaseRepackFactory):
    # Override ignore_dirs so that we don't delete l10n nightly builds
    # before running a l10n nightly build
    ignore_dirs = MozillaBuildFactory.ignore_dirs + ['*-nightly']

    def __init__(self, skipBlankRepos=False, mozRepoPath='',
                 inspectorRepoPath='', venkmanRepoPath='',
                 chatzillaRepoPath='', cvsroot='', buildRevision='',
                 **kwargs):
        self.skipBlankRepos = skipBlankRepos
        self.mozRepoPath = mozRepoPath
        self.inspectorRepoPath = inspectorRepoPath
        self.venkmanRepoPath = venkmanRepoPath
        self.chatzillaRepoPath = chatzillaRepoPath
        self.cvsroot = cvsroot
        self.buildRevision = buildRevision
        BaseRepackFactory.__init__(self, mozillaDir='mozilla',
            mozconfigBranch='default', **kwargs)

    def getSources(self):
        BaseRepackFactory.getSources(self)
        # build up the checkout command with all options
        co_command = ['python', 'client.py', 'checkout',
                      WithProperties('--comm-rev=%(en_revision)s')]
        if self.mozRepoPath:
            co_command.append('--mozilla-repo=%s' % self.getRepository(self.mozRepoPath))
        if self.inspectorRepoPath:
            co_command.append('--inspector-repo=%s' % self.getRepository(self.inspectorRepoPath))
        elif self.skipBlankRepos:
            co_command.append('--skip-inspector')
        if self.venkmanRepoPath:
            co_command.append('--venkman-repo=%s' % self.getRepository(self.venkmanRepoPath))
        elif self.skipBlankRepos:
            co_command.append('--skip-venkman')
        if self.chatzillaRepoPath:
            co_command.append('--chatzilla-repo=%s' % self.getRepository(self.chatzillaRepoPath))
        elif self.skipBlankRepos:
            co_command.append('--skip-chatzilla')
        if self.cvsroot:
            co_command.append('--cvsroot=%s' % self.cvsroot)
        if self.buildRevision:
            co_command.append('--comm-rev=%s' % self.buildRevision)
            co_command.append('--mozilla-rev=%s' % self.buildRevision)
            co_command.append('--inspector-rev=%s' % self.buildRevision)
            co_command.append('--venkman-rev=%s' % self.buildRevision)
            co_command.append('--chatzilla-rev=%s' % self.buildRevision)
        # execute the checkout
        self.addStep(MockCommand(
         command=co_command,
         description=['running', 'client.py', 'checkout'],
         descriptionDone=['client.py', 'checkout'],
         haltOnFailure=True,
         workdir='%s/%s' % (self.baseWorkDir, self.origSrcDir),
         timeout=60*60*3, # 3 hours (crazy, but necessary for now)
         mock=self.use_mock,
         target=self.mock_target,
        ))

class NightlyRepackFactory(BaseRepackFactory, NightlyBuildFactory):
    extraConfigureArgs = []

    def __init__(self, enUSBinaryURL, nightly=False, env={},
                 ausBaseUploadDir=None, updatePlatform=None,
                 downloadBaseURL=None, ausUser=None, ausSshKey=None,
                 ausHost=None, l10nNightlyUpdate=False, l10nDatedDirs=False,
                 createPartial=False, extraConfigureArgs=[], **kwargs):
        self.nightly = nightly
        self.l10nNightlyUpdate = l10nNightlyUpdate
        self.ausBaseUploadDir = ausBaseUploadDir
        self.updatePlatform = updatePlatform
        self.downloadBaseURL = downloadBaseURL
        self.ausUser = ausUser
        self.ausSshKey = ausSshKey
        self.ausHost = ausHost
        self.createPartial = createPartial
        self.geriatricMasters = []
        self.extraConfigureArgs = extraConfigureArgs

        # This is required because this __init__ doesn't call the
        # NightlyBuildFactory __init__ where self.complete_platform
        # is set.  This is only used for android, which doesn't
        # use this factory, so is safe
        self.complete_platform = ''

        env = env.copy()

        env.update({'EN_US_BINARY_URL':enUSBinaryURL})

        # Unfortunately, we can't call BaseRepackFactory.__init__() before this
        # because it needs self.postUploadCmd set
        assert 'project' in kwargs
        assert 'repoPath' in kwargs

        # 1) upload preparation
        if 'branchName' in kwargs:
          uploadDir = '%s-l10n' % kwargs['branchName']
        else:
          uploadDir = '%s-l10n' % self.getRepoName(kwargs['repoPath'])

        uploadArgs = dict(
                product=kwargs['project'],
                branch=uploadDir,
                as_list=False,
                )
        if l10nDatedDirs:
            # nightly repacks and on-change upload to different places
            if self.nightly:
                uploadArgs['buildid'] = WithProperties("%(buildid)s")
                uploadArgs['to_latest'] = True
                uploadArgs['to_dated'] = True
            else:
                # For the repack-on-change scenario we just want to upload
                # to tinderbox builds
                uploadArgs['upload_dir'] = uploadDir
                uploadArgs['to_tinderbox_builds'] = True
        else:
            # for backwards compatibility when the nightly and repack on-change
            # runs were the same 
            uploadArgs['to_latest'] = True

        self.postUploadCmd = postUploadCmdPrefix(**uploadArgs)

        # 2) preparation for updates
        if l10nNightlyUpdate and self.nightly:
            env.update({'MOZ_MAKE_COMPLETE_MAR': '1', 
                        'DOWNLOAD_BASE_URL': '%s/nightly' % self.downloadBaseURL})
            self.extraConfigureArgs += ['--enable-update-packaging']


        BaseRepackFactory.__init__(self, env=env, **kwargs)

        if l10nNightlyUpdate:
            assert ausBaseUploadDir and updatePlatform and downloadBaseURL
            assert ausUser and ausSshKey and ausHost

            # To preserve existing behavior, we need to set the
            # ausFullUploadDir differently for when we are create all the
            # mars (complete+partial) ourselves.
            if self.createPartial:
                # e.g.:
                # /opt/aus2/incoming/2/Firefox/mozilla-central/WINNT_x86-msvc
                self.ausFullUploadDir = '%s/%s' % (self.ausBaseUploadDir,
                                                   self.updatePlatform)
            else:
                # this is a tad ugly because we need python interpolation
                # as well as WithProperties, e.g.:
                # /opt/aus2/build/0/Firefox/mozilla-central/WINNT_x86-msvc/2008010103/en-US
                self.ausFullUploadDir = '%s/%s/%%(buildid)s/%%(locale)s' % \
                  (self.ausBaseUploadDir, self.updatePlatform)
            NightlyBuildFactory.addCreateSnippetsSteps(self,
                                                       milestone_extra='-l10n')
            NightlyBuildFactory.addUploadSnippetsSteps(self)

    def getPreviousBuildUploadDir(self):
        if self.createPartial:
            return "%s/%%(previous_buildid)s/%%(locale)s" % \
                                         self.ausFullUploadDir
        else:
            return self.ausFullUploadDir

    def getCurrentBuildUploadDir(self):
        if self.createPartial:
            return "%s/%%(buildid)s/%%(locale)s" % self.ausFullUploadDir
        else:
            return self.ausFullUploadDir

    def updateSources(self):
        self.addStep(MockCommand(
         name='update_locale_source',
         command=['hg', 'up', '-C', '-r', self.l10nTag],
         description='update workdir',
         workdir=WithProperties('build/l10n/%(locale)s'),
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(SetProperty(
                     command=['hg', 'ident', '-i'],
                     haltOnFailure=True,
                     property='l10n_revision',
                     workdir=WithProperties('build/l10n/%(locale)s')
        ))

    def downloadBuilds(self):
        self.addStep(RetryingMockCommand(
         name='wget_enUS',
         command=self.makeCmd + ['wget-en-US'],
         descriptionDone='wget en-US',
         env=self.env,
         haltOnFailure=True,
         workdir='%s/%s/locales' % (self.absObjDir, self.appName),
         mock=self.use_mock,
         target=self.mock_target,
        ))

    def updateEnUS(self):
        '''Update en-US to the source stamp we get from make ident.

        Requires that we run make unpack first.
        '''
        self.addStep(MockCommand(
                     name='make_unpack',
                     command=self.makeCmd + ['unpack'],
                     descriptionDone='unpacked en-US',
                     haltOnFailure=True,
                     env=self.env,
                     workdir='%s/%s/locales' % (self.absObjDir, self.appName),
                     mock=self.use_mock,
                     target=self.mock_target,
                     ))
        self.addStep(SetProperty(
                     command=self.makeCmd + ['ident'],
                     haltOnFailure=True,
                     env=self.env,
                     workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
                     extract_fn=identToProperties('fx_revision')
        ))
        self.addStep(MockCommand(
                     name='update_enUS_revision',
                     command=['hg', 'update', '-C', '-r',
                              WithProperties('%(fx_revision)s')],
                     haltOnFailure=True,
                     workdir='build/' + self.origSrcDir,
                     mock=self.use_mock,
                     target=self.mock_target,
        ))

    def tinderboxPrintRevisions(self):
        self.tinderboxPrint('fx_revision',WithProperties('%(fx_revision)s'))
        self.tinderboxPrint('l10n_revision',WithProperties('%(l10n_revision)s'))

    def downloadMarTools(self):
        mar = 'mar'
        mbsdiff = 'mbsdiff'
        if self.platform.startswith('win'):
            mar += '.exe'
            mbsdiff += '.exe'
        
        baseURL = 'http://%s' % self.stageServer + \
                  '/pub/mozilla.org/%s' % self.productName + \
                  '/nightly/latest-%s' % self.branchName + \
                  '/mar-tools/%s' % self.platform
        marURL = '%s/%s' % (baseURL, mar)
        mbsdiffURL = '%s/%s' % (baseURL, mbsdiff)
        self.addStep(MockCommand(
            name='get_mar',
            description=['get', 'mar'],
            command=['bash', '-c',
                     '''if [ ! -f %s ]; then
                       wget -O  %s --no-check-certificate %s;
                     fi;
                     (test -e %s && test -s %s) || exit 1;
                     chmod 755 %s'''.replace("\n", "") % (mar, mar, marURL, mar, mar, mar)],
            workdir='%s/dist/host/bin' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='get_mbsdiff',
            description=['get', 'mbsdiff'],
            command=['bash', '-c',
                     '''if [ ! -f %s ]; then
                       wget -O  %s --no-check-certificate %s;
                     fi;
                     (test -e %s && test -s %s) || exit 1;
                     chmod 755 %s'''.replace("\n", "") % (mbsdiff, mbsdiff, mbsdiffURL, mbsdiff, mbsdiff, mbsdiff)],
            workdir='%s/dist/host/bin' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))

    def makePartialTools(self):
        # Build the tools we need for update-packaging, specifically bsdiff.
        # Configure can take a while.
        if self.makeCmd[0] == 'make':
           makeCmd = 'make '
        else:
           # pymake - we can't use self.makeCmd due to embedded WithProperties
           makeCmd = 'python2.7 %(basedir)s/build/build/pymake/make.py '
        # now build the command
        command = 'if [ ! -e dist/host/bin/mbsdiff ]; then ' +\
                  makeCmd + 'tier_base;' + makeCmd + 'tier_nspr; '+\
                  makeCmd + '-C config;' +\
                  makeCmd + '-C modules/libmar;' +\
                  makeCmd + '-C modules/libbz2;' +\
                  makeCmd + '-C other-licenses/bsdiff;' +\
                  'fi'
        self.addStep(MockCommand(
            name='make_bsdiff',
            command=['sh', '-c', WithProperties(command)],
            description=['make', 'bsdiff'],
            env=self.env,
            workdir=self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))

    # The parent class gets us most of the way there, we just need to add the
    # locale.
    def getCompleteMarPatternMatch(self):
        return '.%(locale)s.' + NightlyBuildFactory.getCompleteMarPatternMatch(self)

    def doRepack(self):
        self.downloadMarTools()
        self.addStep(MockCommand(
         name='make_tier_base',
         command=self.makeCmd + ['tier_base'],
         workdir='%s/%s' % (self.baseWorkDir, self.mozillaObjdir),
         description=['make tier_base'],
         env=self.env,
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MockCommand(
         name='make_tier_nspr',
         command=self.makeCmd + ['tier_nspr'],
         workdir='%s/%s' % (self.baseWorkDir, self.mozillaObjdir),
         description=['make tier_nspr'],
         env=self.env,
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        if self.l10nNightlyUpdate:
            # Because we're generating updates we need to build the libmar tools
            self.addStep(MockCommand(
             name='make_libmar',
             command=self.makeCmd,
             env=self.env,
             workdir='%s/%s/modules/libmar' % (self.baseWorkDir, self.mozillaObjdir),
             description=['make', 'modules/libmar'],
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
            ))
        self.addStep(MockCommand(
         name='repack_installers',
         description=['repack', 'installers'],
         command=self.makeCmd + [WithProperties('installers-%(locale)s'),
                                 WithProperties('LOCALE_MERGEDIR=%(basedir)s/' + \
                                                "%s/merged" % self.baseWorkDir)],
         env = self.env,
         haltOnFailure=True,
         workdir='%s/%s/locales' % (self.absObjDir, self.appName),
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(FindFile(
            name='find_inipath',
            filename='application.ini',
            directory='dist/l10n-stage',
            filetype='file',
            max_depth=5,
            property_name='inipath',
            workdir=self.absMozillaObjDir,
            haltOnFailure=True,
        ))
        self.addStep(SetProperty(
            command=['python', 'config/printconfigsetting.py',
                     WithProperties('%(inipath)s'),
                     'App', 'BuildID'],
            property='buildid',
            name='get_build_id',
            workdir=self.absMozillaSrcDir,
        ))
        if self.l10nNightlyUpdate:
            # We need the appVersion to create snippets
            self.addStep(SetProperty(
                command=['python', 'config/printconfigsetting.py',
                         WithProperties('%(inipath)s'),
                         'App', 'Version'],
                property='appVersion',
                name='get_app_version',
                workdir=self.absMozillaSrcDir,
            ))
            self.addFilePropertiesSteps(filename='*.complete.mar',
                                        directory='%s/dist/update' % self.absMozillaSrcDir,
                                        fileType='completeMar',
                                        haltOnFailure=True)

        # Remove the source (en-US) package so as not to confuse later steps
        # that look up build details.
        self.addStep(MockCommand(name='rm_en-US_build',
                                  command=['bash', '-c', 'rm -rvf *.en-US.*'],
                                  description=['remove','en-US','build'],
                                  env=self.env,
                                  workdir='%s/dist' % self.absMozillaObjDir,
                                  haltOnFailure=True,
                                  mock=self.use_mock,
                                  target=self.mock_target,)
         )
        if self.l10nNightlyUpdate and self.createPartial:
            self.addCreatePartialUpdateSteps(extraArgs=[WithProperties('AB_CD=%(locale)s')])


class CCNightlyRepackFactory(CCBaseRepackFactory, NightlyRepackFactory):
    def __init__(self, skipBlankRepos=False, mozRepoPath='',
                 inspectorRepoPath='', venkmanRepoPath='',
                 chatzillaRepoPath='', cvsroot='', buildRevision='',
                 **kwargs):
        self.skipBlankRepos = skipBlankRepos
        self.mozRepoPath = mozRepoPath
        self.inspectorRepoPath = inspectorRepoPath
        self.venkmanRepoPath = venkmanRepoPath
        self.chatzillaRepoPath = chatzillaRepoPath
        self.cvsroot = cvsroot
        self.buildRevision = buildRevision
        NightlyRepackFactory.__init__(self, mozillaDir='mozilla', **kwargs)

    # it sucks to override all of updateEnUS but we need to do it that way
    # this is basically mirroring what mobile does
    def updateEnUS(self):
        '''Update en-US to the source stamp we get from make ident.

        Requires that we run make unpack first.
        '''
        self.addStep(MockCommand(
                     name='make_unpack',
                     command=self.makeCmd + ['unpack'],
                     descriptionDone='unpacked en-US',
                     haltOnFailure=True,
                     env=self.env,
                     workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
                     mock=self.use_mock,
                     target=self.mock_target,
        ))
        
        self.addStep(SetProperty(
                     command=self.makeCmd + ['ident'],
                     haltOnFailure=True,
                     env=self.env,
                     workdir='%s/%s/%s/locales' % (self.baseWorkDir, self.objdir, self.appName),
                     extract_fn=identToProperties()
        ))
        self.addStep(MockCommand(
                     name='update_comm_enUS_revision',
                     command=['hg', 'update', '-C', '-r',
                              WithProperties('%(comm_revision)s')],
                     haltOnFailure=True,
                     workdir='%s/%s' % (self.baseWorkDir, self.origSrcDir),
                     mock=self.use_mock,
                     target=self.mock_target,))
        self.addStep(MockCommand(
                     name='update_mozilla_enUS_revision',
                     command=['hg', 'update', '-C', '-r',
                              WithProperties('%(moz_revision)s')],
                     haltOnFailure=True,
                     workdir='%s/%s' % (self.baseWorkDir, self.mozillaSrcDir),
                     mock=self.use_mock,
                     target=self.mock_target,))

    def tinderboxPrintRevisions(self):
        self.tinderboxPrint('comm_revision',WithProperties('%(comm_revision)s'))
        self.tinderboxPrint('moz_revision',WithProperties('%(moz_revision)s'))
        self.tinderboxPrint('l10n_revision',WithProperties('%(l10n_revision)s'))

    # BaseRepackFactory defines that, and our inheritance chain makes us look
    # there before NightlyRepackFactory, so we need to define it here and call
    # the actually wanted implementation.
    def doRepack(self):
        NightlyRepackFactory.doRepack(self)


class ReleaseFactory(MozillaBuildFactory):
    def getCandidatesDir(self, product, version, buildNumber,
                         nightlyDir="nightly"):
        # can be used with rsync, eg host + ':' + getCandidatesDir()
        # and "http://' + host + getCandidatesDir()
        return '/pub/mozilla.org/' + product + '/' + nightlyDir + '/' + \
               str(version) + '-candidates/build' + str(buildNumber) + '/'

    def getShippedLocales(self, sourceRepo, baseTag, appName):
        return '%s/raw-file/%s_RELEASE/%s/locales/shipped-locales' % \
                 (sourceRepo, baseTag, appName)

    def getSshKeyOption(self, hgSshKey):
        if hgSshKey:
            return '-i %s' % hgSshKey
        return hgSshKey

    def makeLongVersion(self, version):
        version = re.sub('a([0-9]+)$', ' Alpha \\1', version)
        version = re.sub('b([0-9]+)$', ' Beta \\1', version)
        version = re.sub('rc([0-9]+)$', ' RC \\1', version)
        return version

class CCReleaseRepackFactory(CCBaseRepackFactory, ReleaseFactory):
    def __init__(self, platform, buildRevision, version, buildNumber,
                 env={}, brandName=None, mergeLocales=False,
                 mozRepoPath='', inspectorRepoPath='', venkmanRepoPath='',
                 chatzillaRepoPath='', cvsroot='', enUSBinaryURL='', **kwargs):
        self.skipBlankRepos = True
        self.mozRepoPath = mozRepoPath
        self.inspectorRepoPath = inspectorRepoPath
        self.venkmanRepoPath = venkmanRepoPath
        self.chatzillaRepoPath = chatzillaRepoPath
        self.cvsroot = cvsroot
        self.buildRevision = buildRevision
        self.version = version
        self.buildNumber = buildNumber
        if brandName:
            self.brandName = brandName
        else:
            self.brandName = kwargs['project'].capitalize()
 
        env = env.copy()

        env.update({'EN_US_BINARY_URL':enUSBinaryURL})

        # more vars are added in downloadBuilds
        env.update({
            'MOZ_PKG_PRETTYNAMES': '1',
            'MOZ_PKG_VERSION': self.version,
            'MOZ_MAKE_COMPLETE_MAR': '1'
        })

        assert 'project' in kwargs
        # TODO: better place to put this/call this
        self.postUploadCmd = 'post_upload.py ' + \
                             '-p %s ' % kwargs['project'] + \
                             '-v %s ' % self.version + \
                             '-n %s ' % self.buildNumber + \
                             '--release-to-candidates-dir'
        BaseRepackFactory.__init__(self, env=env, platform=platform,
                                   mergeLocales=mergeLocales, mozillaDir='mozilla',
                                   **kwargs)

    # Repeated here since the Parent classes fail hgtool/checkouts due to
    # relbranch issues, and not actually having the tag after clone
    def getSources(self):
        self.addStep(MercurialCloneCommand(
         name='get_enUS_src',
         command=['sh', '-c',
          WithProperties('if [ -d '+self.origSrcDir+'/.hg ]; then ' +
                         'hg -R '+self.origSrcDir+' pull && '+
                         'hg -R '+self.origSrcDir+' up -C ;'+
                         'else ' +
                         'hg clone ' +
                         'http://'+self.hgHost+'/'+self.repoPath+' ' +
                         self.origSrcDir+' ; ' +
                         'fi ' +
                         '&& hg -R '+self.origSrcDir+' update -C -r %(en_revision)s')],
         descriptionDone="en-US source",
         workdir=self.baseWorkDir,
         haltOnFailure=True,
         timeout=30*60 # 30 minutes
        ))
        self.addStep(MercurialCloneCommand(
         name='get_locale_src',
         command=['sh', '-c',
          WithProperties('if [ -d %(locale)s/.hg ]; then ' +
                         'hg -R %(locale)s pull -r default ; ' +
                         'else ' +
                         'hg clone ' +
                         'http://'+self.hgHost+'/'+self.l10nRepoPath+ 
                           '/%(locale)s/ ; ' +
                         'fi ' +
                         '&& hg -R %(locale)s update -C -r %(l10n_revision)s')],
         descriptionDone="locale source",
         timeout=10*60, # 10 minutes
         haltOnFailure=True,
         workdir='%s/l10n' % self.baseWorkDir
        ))
        # build up the checkout command with all options
        co_command = ['python', 'client.py', 'checkout',
                      WithProperties('--comm-rev=%(en_revision)s')]
        if self.mozRepoPath:
            co_command.append('--mozilla-repo=%s' % self.getRepository(self.mozRepoPath))
        if self.inspectorRepoPath:
            co_command.append('--inspector-repo=%s' % self.getRepository(self.inspectorRepoPath))
        elif self.skipBlankRepos:
            co_command.append('--skip-inspector')
        if self.venkmanRepoPath:
            co_command.append('--venkman-repo=%s' % self.getRepository(self.venkmanRepoPath))
        elif self.skipBlankRepos:
            co_command.append('--skip-venkman')
        if self.chatzillaRepoPath:
            co_command.append('--chatzilla-repo=%s' % self.getRepository(self.chatzillaRepoPath))
        elif self.skipBlankRepos:
            co_command.append('--skip-chatzilla')
        if self.cvsroot:
            co_command.append('--cvsroot=%s' % self.cvsroot)
        if self.buildRevision:
            co_command.append('--comm-rev=%s' % self.buildRevision)
            co_command.append('--mozilla-rev=%s' % self.buildRevision)
            co_command.append('--inspector-rev=%s' % self.buildRevision)
            co_command.append('--venkman-rev=%s' % self.buildRevision)
            co_command.append('--chatzilla-rev=%s' % self.buildRevision)
        # execute the checkout
        self.addStep(MockCommand(
         command=co_command,
         description=['running', 'client.py', 'checkout'],
         descriptionDone=['client.py', 'checkout'],
         haltOnFailure=True,
         workdir='%s/%s' % (self.baseWorkDir, self.origSrcDir),
         timeout=60*60*3, # 3 hours (crazy, but necessary for now)
         mock=self.use_mock,
         target=self.mock_target,
        ))

    def updateSources(self):
        self.addStep(MockCommand(
         name='update_sources',
         command=['hg', 'up', '-C', '-r', self.buildRevision],
         workdir='build/'+self.origSrcDir,
         description=['update %s' % self.branchName,
                      'to %s' % self.buildRevision],
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MockCommand(
         name='update_locale_sources',
         command=['hg', 'up', '-C', '-r', self.buildRevision],
         workdir=WithProperties('build/l10n/%(locale)s'),
         description=['update to', self.buildRevision],
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(SetProperty(
                     command=['hg', 'ident', '-i'],
                     haltOnFailure=True,
                     property='l10n_revision',
                     workdir=WithProperties('build/l10n/%(locale)s')
        ))
        self.addStep(MockCommand(
         command=['hg', 'up', '-C', '-r', self.buildRevision],
         workdir='build/'+self.mozillaSrcDir,
         description=['update mozilla',
                      'to %s' % self.buildRevision],
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        if self.venkmanRepoPath:
            self.addStep(MockCommand(
             command=['hg', 'up', '-C', '-r', self.buildRevision],
             workdir='build/'+self.mozillaSrcDir+'/extensions/venkman',
             description=['update venkman',
                          'to %s' % self.buildRevision],
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
            ))
        if self.inspectorRepoPath:
            self.addStep(MockCommand(
             command=['hg', 'up', '-C', '-r', self.buildRevision],
             workdir='build/'+self.mozillaSrcDir+'/extensions/inspector',
             description=['update inspector',
                          'to %s' % self.buildRevision],
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
            ))
        if self.chatzillaRepoPath:
            self.addStep(MockCommand(
             command=['hg', 'up', '-C', '-r', self.buildRevision],
             workdir='build/'+self.mozillaSrcDir+'/extensions/irc',
             description=['update chatzilla',
                          'to %s' % self.buildRevision],
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
            ))

    def preClean(self):
        # We need to know the absolute path to the input builds when we repack,
        # so we need retrieve at run-time as a build property
        self.addStep(SetProperty(
         command=['bash', '-c', 'pwd'],
         property='srcdir',
         workdir='build/'+self.origSrcDir
        ))

    def downloadBuilds(self):
        if self.mozillaDir:
            workdir = '%s/%s/locales' % (self.absObjDir,
                                         self.appName)
        else:
            workdir = '%s/%s/locales' % (self.absMozillaObjDir,
                                         self.appName)

        self.addStep(MockCommand(
                     name='wget_enUS',
                     command=self.makeCmd + ['wget-en-US'],
                     descriptionDone='wget en-US',
                     env=self.env,
                     haltOnFailure=True,
                     workdir=workdir,
                     mock=self.use_mock,
                     target=self.mock_target,
                     ))

        # Break this function early, since I'm testing
        return
        candidatesDir = 'http://%s' % self.stageServer + \
                        '/pub/mozilla.org/%s/nightly' % self.project + \
                        '/%s-candidates/build%s' % (self.version,
                                                    self.buildNumber)

        # This block sets platform specific data that our wget command needs.
        #  build is mapping between the local and remote filenames
        #  platformDir is the platform specific directory builds are contained
        #    in on the stagingServer.
        # This block also sets the necessary environment variables that the
        # doRepack() steps rely on to locate their source build.
        builds = {}
        platformDir = getPlatformFtpDir(self.platform.split("-")[0])
        if self.platform.startswith('linux'):
            filename = '%s.tar.bz2' % self.project
            builds[filename] = '%s-%s.tar.bz2' % (self.project, self.version)
            self.env['ZIP_IN'] = WithProperties('%(srcdir)s/' + filename)
        elif self.platform.startswith('macosx'):
            filename = '%s.dmg' % self.project
            builds[filename] = '%s %s.dmg' % (self.brandName,
                                              self.version)
            self.env['ZIP_IN'] = WithProperties('%(srcdir)s/' + filename)
        elif self.platform.startswith('win32'):
            platformDir = 'unsigned/' + platformDir
            filename = '%s.zip' % self.project
            instname = '%s.exe' % self.project
            builds[filename] = '%s-%s.zip' % (self.project, self.version)
            builds[instname] = '%s Setup %s.exe' % (self.brandName,
                                                    self.version)
            self.env['ZIP_IN'] = WithProperties('%(srcdir)s/' + filename)
            self.env['WIN32_INSTALLER_IN'] = \
              WithProperties('%(srcdir)s/' + instname)
        else:
            raise "Unsupported platform"

        for name in builds:
            self.addStep(MockCommand(
             name='get_candidates_%s' % name,
             command=['wget', '-O', name, '--no-check-certificate',
                      '%s/%s/en-US/%s' % (candidatesDir, platformDir,
                                          builds[name])],
             workdir='build/'+self.origSrcDir,
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
            ))

    def downloadMarTools(self):
        mar = 'mar'
        mbsdiff = 'mbsdiff'
        if self.platform.startswith('win'):
            mar += '.exe'
            mbsdiff += '.exe'

        baseURL = 'http://%s' % self.stageServer + \
                  '/pub/mozilla.org/%s/candidates' % self.project + \
                  '/%s-candidates/build%s' % (self.version, self.buildNumber) + \
                  '/mar-tools/%s' % self.platform
        marURL = '%s/%s' % (baseURL, mar)
        mbsdiffURL = '%s/%s' % (baseURL, mbsdiff)
        self.addStep(MockCommand(
            name='get_mar',
            description=['get', 'mar'],
            command=['bash', '-c',
                     '''if [ ! -f %s ]; then
                       wget -O  %s --no-check-certificate %s;
                     fi;
                     (test -e %s && test -s %s) || exit 1;
                     chmod 755 %s'''.replace("\n", "") % (mar, mar, marURL, mar, mar, mar)],
            workdir='%s/dist/host/bin' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))
        self.addStep(MockCommand(
            name='get_mbsdiff',
            description=['get', 'mbsdiff'],
            command=['bash', '-c',
                     '''if [ ! -f %s ]; then
                       wget -O  %s --no-check-certificate %s;
                     fi;
                     (test -e %s && test -s %s) || exit 1;
                     chmod 755 %s'''.replace("\n", "") % (mbsdiff, mbsdiff, mbsdiffURL, mbsdiff, mbsdiff, mbsdiff)],
            workdir='%s/dist/host/bin' % self.absMozillaObjDir,
            haltOnFailure=True,
            mock=self.use_mock,
            target=self.mock_target,
        ))

    def doRepack(self):
        self.downloadMarTools()

        self.addStep(MockCommand(
         name='repack_installers',
         description=['repack', 'installers'],
         command=self.makeCmd + [WithProperties('installers-%(locale)s'),
                                 WithProperties('LOCALE_MERGEDIR=%(basedir)s/' + \
                                                "%s/merged" % self.baseWorkDir)],
         env=self.env,
         haltOnFailure=True,
         workdir='%s/%s/locales' % (self.absObjDir, self.appName),
         mock=self.use_mock,
         target=self.mock_target,
        ))

class StagingRepositorySetupFactory(ReleaseFactory):
    """This Factory should be run at the start of a staging release run. It
       deletes and reclones all of the repositories in 'repositories'. Note that
       the staging buildTools repository should _not_ be recloned, as it is
       used by many other builders, too.
    """
    def __init__(self, username, sshKey, repositories, userRepoRoot,
                 **kwargs):
        # MozillaBuildFactory needs the 'repoPath' argument, but we don't
        ReleaseFactory.__init__(self, repoPath='nothing', **kwargs)
        for repoPath in sorted(repositories.keys()):
            repo = self.getRepository(repoPath)
            repoName = self.getRepoName(repoPath)
            # Don't use cache for user repos
            rnd = random.randint(100000, 999999)
            userRepoURL = '%s/%s?rnd=%s' % (self.getRepository(userRepoRoot),
                                        repoName, rnd)

            # test for existence
            command = 'wget -O /dev/null %s' % repo
            command += ' && { '
            command += 'if wget -q -O /dev/null %s; then ' % userRepoURL
            # if it exists, delete it
            command += 'echo "Deleting %s"; ' % repoName
            command += 'ssh -l %s -i %s %s edit %s delete YES; ' % \
              (username, sshKey, self.hgHost, repoName)
            command += 'else echo "Not deleting %s"; exit 0; fi }' % repoName

            self.addStep(MockCommand(
             name='delete_repo',
             command=['bash', '-c', command],
             description=['delete', repoName],
             haltOnFailure=True,
             timeout=30*60, # 30 minutes
             mock=self.use_mock,
             target=self.mock_target,
            ))

        # Wait for hg.m.o to catch up
        self.addStep(MockCommand(
         name='wait_for_hg',
         command=['sleep', '600'],
         description=['wait', 'for', 'hg'],
         mock=self.use_mock,
         target=self.mock_target,
        ))

        for repoPath in sorted(repositories.keys()):
            repo = self.getRepository(repoPath)
            repoName = self.getRepoName(repoPath)
            timeout = 60*60
            command = ['python',
                       WithProperties('%(toolsdir)s/buildfarm/utils/retry.py'),
                       '--timeout', timeout,
                       'ssh', '-l', username, '-oIdentityFile=%s' % sshKey,
                       self.hgHost, 'clone', repoName, repoPath]

            self.addStep(MockCommand(
             name='recreate_repo',
             command=command,
             description=['recreate', repoName],
             timeout=timeout,
             mock=self.use_mock,
             target=self.mock_target,
            ))

        # Wait for hg.m.o to catch up
        self.addStep(MockCommand(
         name='wait_for_hg',
         command=['sleep', '600'],
         description=['wait', 'for', 'hg'],
         mock=self.use_mock,
         target=self.mock_target,
        ))



class ReleaseTaggingFactory(ReleaseFactory):
    def __init__(self, repositories, productName, appName, version, appVersion,
                 milestone, baseTag, buildNumber, hgUsername, hgSshKey=None,
                 relbranchPrefix=None, buildSpace=1.5, **kwargs):
        """Repositories looks like this:
            repositories[name]['revision']: changeset# or tag
            repositories[name]['relbranchOverride']: branch name
            repositories[name]['bumpFiles']: [filesToBump]
           eg:
            repositories['http://hg.mozilla.org/mozilla-central']['revision']:
              d6a0a4fca081
            repositories['http://hg.mozilla.org/mozilla-central']['relbranchOverride']:
              GECKO191_20080828_RELBRANCH
            repositories['http://hg.mozilla.org/mozilla-central']['bumpFiles']:
              ['client.mk', 'browser/config/version.txt',
               'js/src/config/milestone.txt', 'config/milestone.txt']
            relbranchOverride is typically used in two situations:
             1) During a respin (buildNumber > 1) when the "release" branch has
                already been created (during build1). In cases like this all
                repositories should have the relbranch specified
             2) During non-Firefox builds. Because Seamonkey, Thunderbird, etc.
                are releases off of the same platform code as Firefox, the
                "release branch" will already exist in mozilla-central but not
                comm-central, mobile-browser, domi, etc. In cases like this,
                mozilla-central and l10n should be specified with a
                relbranchOverride and the other source repositories should NOT
                specify one.
           productName: The name of the actual *product* being shipped.
                        Examples include: firefox, thunderbird, seamonkey.
                        This is only used for the automated check-in message
                        the version bump generates.
           appName: The "application" name (NOT product name). Examples:
                    browser, suite, mailnews. It is used in version bumping
                    code and assumed to be a subdirectory of the source
                    repository being bumped. Eg, for Firefox, appName should be
                    'browser', which is a subdirectory of 'mozilla-central'.
                    For Thunderbird, it would be 'mailnews', a subdirectory
                    of 'comm-central'.
           version: What this build is actually called. I most cases this is
                    the version number of the application, eg, 3.0.6, 3.1b2.
                    During the RC phase we "call" builds, eg, 3.1 RC1, but the
                    version of the application is still 3.1. In these cases,
                    version should be set to, eg, 3.1rc1.
           appVersion: The current version number of the application being
                       built. Eg, 3.0.2 for Firefox, 2.0 for Seamonkey, etc.
                       This is different than the platform version. See below.
                       This is usually the same as 'version', except during the
                       RC phase. Eg, when version is 3.1rc1 appVersion is still
                       3.1.
           milestone: The current version of *Gecko*. This is generally
                      along the lines of: 1.8.1.14, 1.9.0.2, etc.
           baseTag: The prefix to use for BUILD/RELEASE tags. It will be
                    post-fixed with _BUILD$buildNumber and _RELEASE. Generally,
                    this is something like: FIREFOX_3_0_2.
           buildNumber: The current build number. If this is the first time
                        attempting a release this is 1. Other times it may be
                        higher. It is used for post-fixing tags and some
                        internal logic.
           hgUsername: The username to use when pushing changes to the remote
                       repository.
           hgSshKey: The full path to the ssh key to use (if necessary) when
                     pushing changes to the remote repository.
           relbranchPrefix: the prefix to start relelease branch names with
                            (defaults to 'GECKO')

        """
        # MozillaBuildFactory needs the 'repoPath' argument, but we don't
        ReleaseFactory.__init__(self, repoPath='nothing', buildSpace=buildSpace,
                                **kwargs)

        # extremely basic validation, to catch really dumb configurations
        assert len(repositories) > 0, \
          'You must provide at least one repository.'
        assert productName, 'You must provide a product name (eg. firefox).'
        assert appName, 'You must provide an application name (eg. browser).'
        assert version, \
          'You must provide an application version (eg. 3.0.2).'
        assert milestone, 'You must provide a milestone (eg. 1.9.0.2).'
        assert baseTag, 'You must provide a baseTag (eg. FIREFOX_3_0_2).'
        assert buildNumber, 'You must provide a buildNumber.'

        # if we're doing a respin we already have a relbranch created
        if buildNumber > 1:
            for repo in repositories:
                assert repositories[repo]['relbranchOverride'], \
                  'No relbranchOverride specified for ' + repo + \
                  '. You must provide a relbranchOverride when buildNumber > 1'

        # now, down to work
        self.buildTag = '%s_BUILD%s' % (baseTag, str(buildNumber))
        self.releaseTag = '%s_RELEASE' % baseTag

        # generate the release branch name, which is based on the
        # version and the current date.
        # looks like: GECKO191_20080728_RELBRANCH
        # This can be overridden per-repository. This case is handled
        # in the loop below
        if not relbranchPrefix:
            relbranchPrefix = 'GECKO'
        relbranchName = '%s%s_%s_RELBRANCH' % (
          relbranchPrefix, milestone.replace('.', ''),
          datetime.now().strftime('%Y%m%d'))

        for repoPath in sorted(repositories.keys()):
            repoName = self.getRepoName(repoPath)
            repo = self.getRepository(repoPath)
            pushRepo = self.getRepository(repoPath, push=True)

            sshKeyOption = self.getSshKeyOption(hgSshKey)

            repoRevision = repositories[repoPath]['revision']
            bumpFiles = repositories[repoPath]['bumpFiles']

            # use repo-specific variable so that a changed name doesn't
            # propagate to later repos without an override
            relbranchOverride = False
            if repositories[repoPath]['relbranchOverride']:
                relbranchOverride = True
                repoRelbranchName = repositories[repoPath]['relbranchOverride']
            else:
                repoRelbranchName = relbranchName

            # For l10n we never bump any files, so this will never get
            # overridden. For source repos, we will do a version bump in build1
            # which we commit, and set this property again, so we tag
            # the right revision. For build2, we don't version bump, and this
            # will not get overridden
            self.addStep(SetBuildProperty(
             property_name="%s-revision" % repoName,
             value=repoRevision,
             haltOnFailure=True
            ))
            # 'hg clone -r' breaks in the respin case because the cloned
            # repository will not have ANY changesets from the release branch
            # and 'hg up -C' will fail
            self.addStep(MercurialCloneCommand(
             name='hg_clone',
             command=['hg', 'clone', repo, repoName],
             workdir='.',
             description=['clone %s' % repoName],
             haltOnFailure=True,
             timeout=30*60 # 30 minutes
            ))
            # for build1 we need to create a branch
            if buildNumber == 1 and not relbranchOverride:
                # remember:
                # 'branch' in Mercurial does not actually create a new branch,
                # it switches the "current" branch to the one of the given name.
                # when buildNumber == 1 this will end up creating a new branch
                # when we commit version bumps and tags.
                # note: we don't actually have to switch to the release branch
                # to create tags, but it seems like a more sensible place to
                # have those commits
                self.addStep(MockCommand(
                 name='hg_update',
                 command=['hg', 'up', '-C', '-r',
                          WithProperties('%s', '%s-revision' % repoName)],
                 workdir=repoName,
                 description=['update', repoName],
                 haltOnFailure=True,
                 mock=self.use_mock,
                 target=self.mock_target,
                ))

                self.addStep(SetProperty(
                 command=['sh', '-c', 'hg branches | grep %s | wc -l' % repoRelbranchName],
                 property='branch_match_count',
                 workdir=repoName,
                 haltOnFailure=True,
                ))

                self.addStep(MockCommand(
                 name='hg_branch',
                 command=['hg', 'branch', repoRelbranchName],
                 workdir=repoName,
                 description=['branch %s' % repoName],
                 doStepIf=lambda step: int(step.getProperty('branch_match_count')) == 0,
                 haltOnFailure=True,
                 mock=self.use_mock,
                 target=self.mock_target,
                ))

                self.addStep(MockCommand(
                 name='switch_branch',
                 command=['hg', 'up', '-C', repoRelbranchName],
                 workdir=repoName,
                 description=['switch to', repoRelbranchName],
                 doStepIf=lambda step: int(step.getProperty('branch_match_count')) > 0,
                 haltOnFailure=True,
                 mock=self.use_mock,
                 target=self.mock_target,
                ))
            # if buildNumber > 1 we need to switch to it with 'hg up -C'
            else:
                self.addStep(MockCommand(
                 name='switch_branch',
                 command=['hg', 'up', '-C', repoRelbranchName],
                 workdir=repoName,
                 description=['switch to', repoRelbranchName],
                 haltOnFailure=True,
                 mock=self.use_mock,
                 target=self.mock_target,
                ))
            # we don't need to do any version bumping if this is a respin
            if buildNumber == 1 and len(bumpFiles) > 0:
                command = ['perl', 'tools/scripts/release/version-bump.pl',
                           '-w', repoName, '-a', appName,
                           '-v', appVersion, '-m', milestone]
                command.extend(bumpFiles)
                self.addStep(MockCommand(
                 name='bump',
                 command=command,
                 workdir='.',
                 description=['bump %s' % repoName],
                 haltOnFailure=True,
                 mock=self.use_mock,
                 target=self.mock_target,
                ))
                self.addStep(MockCommand(
                 name='hg_diff',
                 command=['hg', 'diff'],
                 workdir=repoName,
                 mock=self.use_mock,
                 target=self.mock_target,
                ))
                self.addStep(MockCommand(
                 # mozilla-central and other developer repositories have a
                 # 'CLOSED TREE' or 'APPROVAL REQUIRED' hook on them which
                 # rejects commits when the tree is declared closed/approval
                 # required. It is very common for us to tag
                 # and branch when the tree is in this state. Adding the
                 # 'CLOSED TREE a=release' string at the end will force the
                 # hook to let us commit regardless of the tree state.
                 name='hg_commit',
                 command=['hg', 'commit', '-u', hgUsername, '-m',
                          'Automated checkin: version bump remove "pre" ' + \
                          ' from version number for ' + productName + ' ' + \
                          version + ' release on ' + repoRelbranchName + ' ' + \
                          'CLOSED TREE a=release'],
                 workdir=repoName,
                 description=['commit %s' % repoName],
                 haltOnFailure=True,
                 mock=self.use_mock,
                 target=self.mock_target,
                ))
                self.addStep(SetProperty(
                 command=['hg', 'identify', '-i'],
                 property='%s-revision' % repoName,
                 workdir=repoName,
                 haltOnFailure=True
                ))
            for tag in (self.buildTag, self.releaseTag):
                self.addStep(MockCommand(
                 name='hg_tag',
                 command=['hg', 'tag', '-u', hgUsername, '-f', '-r',
                          WithProperties('%s', '%s-revision' % repoName),
                          '-m',
                          # This part is pretty ugly. Because we need both
                          # WithProperties interpolation (for repoName-revision)
                          # and regular variables we need to piece it together
                          # this way.
                          WithProperties('Added tag ' + tag + \
                            ' for changeset ' + \
                            '%(' + repoName + '-revision' + ')s. ' + \
                            'CLOSED TREE a=release'),
                          tag],
                 workdir=repoName,
                 description=['tag %s' % repoName],
                 haltOnFailure=True,
                 mock=self.use_mock,
                 target=self.mock_target,
                ))
            self.addStep(MockCommand(
             name='hg_out',
             command=['hg', 'out', '-e',
                      'ssh -l %s %s' % (hgUsername, sshKeyOption),
                      pushRepo],
             workdir=repoName,
             description=['hg out', repoName],
             mock=self.use_mock,
             target=self.mock_target,
            ))
            self.addStep(MockCommand(
             name='hg_push',
             command=['hg', 'push', '-e',
                      'ssh -l %s %s' % (hgUsername, sshKeyOption),
                      '-f', pushRepo],
             workdir=repoName,
             description=['push %s' % repoName],
             haltOnFailure=True,
             mock=self.use_mock,
             target=self.mock_target,
            ))


class SingleSourceFactory(ReleaseFactory):
    def __init__(self, productName, version, baseTag, stagingServer,
                 stageUsername, stageSshKey, buildNumber, mozconfig,
                 configRepoPath, configSubDir, objdir='',
                 mozillaDir=None, autoconfDirs=['.'], buildSpace=1,
                 mozconfigBranch="production", appVersion=None, **kwargs):
        ReleaseFactory.__init__(self, buildSpace=buildSpace, **kwargs)

        self.mozconfig = mozconfig
        self.configRepoPath=configRepoPath
        self.configSubDir=configSubDir
        self.mozconfigBranch = mozconfigBranch
        self.releaseTag = '%s_RELEASE' % (baseTag)
        self.bundleFile = 'source/%s-%s.bundle' % (productName, version)
        self.sourceTarball = 'source/%s-%s.source.tar.bz2' % (productName, version)

        self.origSrcDir = self.branchName

        # Mozilla subdir
        if mozillaDir:
            self.mozillaDir = '/%s' % mozillaDir
            self.mozillaSrcDir = '%s/%s' % (self.origSrcDir, mozillaDir)
        else:
            self.mozillaDir = ''
            self.mozillaSrcDir = self.origSrcDir

        # self.mozillaObjdir is used in SeaMonkey's and Thunderbird's case
        self.objdir = objdir or self.origSrcDir
        self.mozillaObjdir = '%s%s' % (self.objdir, self.mozillaDir)
        self.distDir = "%s/dist" % self.mozillaObjdir

        # Make sure MOZ_PKG_PRETTYNAMES is set so that our source package is
        # created in the expected place.
        self.env['MOZ_OBJDIR'] = self.objdir
        self.env['MOZ_PKG_PRETTYNAMES'] = '1'
        if appVersion is None or version != appVersion:
            self.env['MOZ_PKG_VERSION'] = version
        self.env['MOZ_PKG_APPNAME'] = productName

        # '-c' is for "release to candidates dir"
        postUploadCmd = 'post_upload.py -p %s -v %s -n %s -c' % \
          (productName, version, buildNumber)
        if productName == 'fennec':
            postUploadCmd = 'post_upload.py -p mobile --nightly-dir candidates -v %s -n %s -c' % \
                          (version, buildNumber)
        uploadEnv = {'UPLOAD_HOST': stagingServer,
                     'UPLOAD_USER': stageUsername,
                     'UPLOAD_SSH_KEY': '~/.ssh/%s' % stageSshKey,
                     'UPLOAD_TO_TEMP': '1',
                     'POST_UPLOAD_CMD': postUploadCmd}

        self.addStep(MockCommand(
         name='rm_srcdir',
         command=['rm', '-rf', 'source'],
         workdir='.',
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MockCommand(
         name='make_srcdir',
         command=['mkdir', 'source'],
         workdir='.',
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MercurialCloneCommand(
         name='hg_clone',
         command=['hg', 'clone', self.repository, self.branchName],
         workdir='.',
         description=['clone %s' % self.branchName],
         haltOnFailure=True,
         timeout=30*60 # 30 minutes
        ))
        # This will get us to the version we're building the release with
        self.addStep(MockCommand(
         name='hg_update',
         command=['hg', 'up', '-C', '-r', self.releaseTag],
         workdir=self.mozillaSrcDir,
         description=['update to', self.releaseTag],
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        # ...And this will get us the tags so people can do things like
        # 'hg up -r FIREFOX_3_1b1_RELEASE' with the bundle
        self.addStep(MockCommand(
         name='hg_update_incl_tags',
         command=['hg', 'up', '-C'],
         workdir=self.mozillaSrcDir,
         description=['update to', 'include tag revs'],
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(SetProperty(
         name='hg_ident_revision',
         command=['hg', 'identify', '-i'],
         property='revision',
         workdir=self.mozillaSrcDir,
         haltOnFailure=True
        ))
        self.addStep(MockCommand(
         name='create_bundle',
         command=['hg', '-R', self.branchName, 'bundle', '--base', 'null',
                  '-r', WithProperties('%(revision)s'),
                  self.bundleFile],
         workdir='.',
         description=['create bundle'],
         haltOnFailure=True,
         timeout=30*60, # 30 minutes
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MockCommand(
         name='delete_metadata',
         command=['rm', '-rf', '.hg'],
         workdir=self.mozillaSrcDir,
         description=['delete metadata'],
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addConfigSteps(workdir=self.mozillaSrcDir)
        self.addStep(MockCommand(
         name='configure',
         command=self.makeCmd + ['-f', 'client.mk', 'configure'],
         workdir=self.mozillaSrcDir,
         env=self.env,
         description=['configure'],
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MockCommand(
         name='make_source-package',
         command=self.makeCmd + ['source-package'],
         workdir="%s/%s" % (self.mozillaSrcDir, self.mozillaObjdir),
         env=self.env,
         description=['make source-package'],
         haltOnFailure=True,
         timeout=30*60, # 30 minutes
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MockCommand(
         name='mv_source-package',
         command=['mv','%s/%s/%s' % (self.branchName,
                                     self.distDir,
                                     self.sourceTarball),
                  self.sourceTarball],
         workdir=".",
         env=self.env,
         description=['mv source-package'],
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        files = [self.bundleFile, self.sourceTarball]
        if self.enableSigning and self.signingServers:
            self.addGetTokenSteps()
            # use a copy of files variable to prevent endless loops
            for f in list(files):
                signingcmd = WithProperties(
                    '%s --formats gpg "%s"' % (self.signing_command, f))
                self.addStep(ShellCommand(
                    name='sign_file',
                    command=signingcmd,
                    workdir='.',
                    env=self.env,
                    description=['sign', f],
                    haltOnFailure=True,
                ))
                files.append('%s.asc' % f)
        self.addStep(RetryingMockCommand(
         name='upload_files',
         command=['python', '%s/build/upload.py' % self.branchName,
                  '--base-path', '.'] + files,
         workdir='.',
         env=uploadEnv,
         description=['upload files'],
         mock=self.use_mock,
         target=self.mock_target,
        ))

    def addConfigSteps(self, workdir='build'):
        assert self.configRepoPath is not None
        assert self.configSubDir is not None
        assert self.mozconfig is not None
        configRepo = self.getRepository(self.configRepoPath)

        self.mozconfig = 'configs/%s/%s/mozconfig' % (self.configSubDir,
                                                      self.mozconfig)
        self.addStep(MockCommand(
                     name='rm_configs',
                     command=['rm', '-rf', 'configs'],
                     description=['removing', 'configs'],
                     descriptionDone=['remove', 'configs'],
                     haltOnFailure=True,
                     workdir='.',
                     mock=self.use_mock,
                     target=self.mock_target,
        ))
        self.addStep(MercurialCloneCommand(
                     name='hg_clone_configs',
                     command=['hg', 'clone', configRepo, 'configs'],
                     description=['checking', 'out', 'configs'],
                     descriptionDone=['checkout', 'configs'],
                     haltOnFailure=True,
                     workdir='.'
        ))
        self.addStep(MockCommand(
                     name='hg_update',
                     command=['hg', 'update', '-r', self.mozconfigBranch],
                     description=['updating', 'mozconfigs'],
                     haltOnFailure=True,
                     workdir='./configs',
                     mock=self.use_mock,
                     target=self.mock_target,
        ))
        self.addStep(MockCommand(
                     # cp configs/mozilla2/$platform/$repo/$type/mozconfig .mozconfig
                     name='cp_mozconfig',
                     command=['cp', self.mozconfig, '%s/.mozconfig' % workdir],
                     description=['copying', 'mozconfig'],
                     descriptionDone=['copy', 'mozconfig'],
                     haltOnFailure=True,
                     workdir='.',
                     mock=self.use_mock,
                     target=self.mock_target,
        ))
        self.addStep(MockCommand(
                     name='cat_mozconfig',
                     command=['cat', '.mozconfig'],
                     workdir=workdir,
                     mock=self.use_mock,
                     target=self.mock_target,
        ))

class MultiSourceFactory(ReleaseFactory):
    """You need to pass in a repoConfig, which will be a list that
       looks like this:
       repoConfig = [{
           'repoPath': repoPath,
           'location': branchName,
           'bundleName': '%s-%s.bundle' % (productName, version)
       }]"""
    def __init__(self, productName, version, baseTag, stagingServer,
                 stageUsername, stageSshKey, buildNumber, autoconfDirs=['.'],
                 buildSpace=1, repoConfig=None, uploadProductName=None,
                 stageNightlyDir="nightly", **kwargs):
        ReleaseFactory.__init__(self, buildSpace=buildSpace, **kwargs)
        releaseTag = '%s_RELEASE' % (baseTag)
        bundleFiles = []
        sourceTarball = 'source/%s-%s.source.tar.bz2' % (productName,
                                                         version)
        if not uploadProductName:
            uploadProductName = productName

        assert repoConfig
        # '-c' is for "release to candidates dir"
        postUploadCmd = 'post_upload.py -p %s -v %s -n %s -c --nightly-dir %s' % \
          (uploadProductName, version, buildNumber, stageNightlyDir)
        uploadEnv = {'UPLOAD_HOST': stagingServer,
                     'UPLOAD_USER': stageUsername,
                     'UPLOAD_SSH_KEY': '~/.ssh/%s' % stageSshKey,
                     'UPLOAD_TO_TEMP': '1',
                     'POST_UPLOAD_CMD': postUploadCmd}

        self.addStep(MockCommand(
         name='rm_srcdir',
         command=['rm', '-rf', 'source'],
         workdir='.',
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        self.addStep(MockCommand(
         name='make_srcdir',
         command=['mkdir', 'source'],
         workdir='.',
         haltOnFailure=True,
         mock=self.use_mock,
         target=self.mock_target,
        ))
        for repo in repoConfig:
            repository = self.getRepository(repo['repoPath'])
            location = repo['location']
            bundleFiles.append('source/%s' % repo['bundleName'