talos-staging/perfrunner.py
author Chris AtLee <catlee@mozilla.com>
Tue, 11 Aug 2009 07:51:11 -0400
changeset 1419 ed1259e43959fe3e43d51f2442ba0d1124ee8edc
parent 1248 ce8bec434f9f4edadc305fe22af849c7513ae21f
permissions -rw-r--r--
Bug 509280 - Add configs for talos staging try using TryTalosFactory. r=anodelman

# -*- Python -*-

from twisted.python import log
from buildbot import buildset
from buildbot.scheduler import Scheduler
from buildbot.process.buildstep import BuildStep
from buildbot.process.factory import BuildFactory
from buildbot.buildset import BuildSet
from buildbot.sourcestamp import SourceStamp
from buildbot.steps.shell import ShellCommand, WithProperties
from buildbot.steps.transfer import FileDownload
from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION
import re, urllib, sys, os
from time import strptime, strftime, localtime
from datetime import datetime
import copy

MozillaEnvironments = { }

# platform SDK location.  we can build both from one generic template.
# modified from vc8 environment
MozillaEnvironments['vc8perf'] = {
    "MOZ_CRASHREPORTER_NO_REPORT": '1',
    "MOZ_NO_REMOTE": '1',
    "NO_EM_RESTART": '1',
    "XPCOM_DEBUG_BREAK": 'warn',
    "CYGWINBASE": 'C:\\cygwin',
    "PATH": 'C:\\Python24;' + \
            'C:\\Python24\\Scripts;' + \
            'C:\\cygwin\\bin;' + \
            'C:\\WINDOWS\\System32;' + \
            'C:\\program files\\gnuwin32\\bin;' + \
            'C:\\WINDOWS;'
}

MozillaEnvironments['linux'] = {
    "MOZ_CRASHREPORTER_NO_REPORT": '1',
    "MOZ_NO_REMOTE": '1',
    "NO_EM_RESTART": '1',
    "XPCOM_DEBUG_BREAK": 'warn',
    "DISPLAY": ":0",
}

MozillaEnvironments['mac'] = {
    "MOZ_NO_REMOTE": '1',
    "NO_EM_RESTART": '1',
    "XPCOM_DEBUG_BREAK": 'warn',
    "MOZ_CRASHREPORTER_NO_REPORT": '1',
    # for extracting dmg's
    "PAGER": '/bin/cat',
}

class MultiBuildScheduler(Scheduler):
    """Trigger N (default three) build requests based upon the same change request"""
    def __init__(self, numberOfBuildsToTrigger=3, **kwargs):
        self.numberOfBuildsToTrigger = numberOfBuildsToTrigger
        Scheduler.__init__(self, **kwargs)

    def fireTimer(self):
        self.timer = None
        self.nextBuildTime = None
        changes = self.importantChanges + self.unimportantChanges
        self.importantChanges = []
        self.unimportantChanges = []

        # submit
        for i in range(0, self.numberOfBuildsToTrigger):
            ss = UnmergableSourceStamp(changes=changes)
            bs = buildset.BuildSet(self.builderNames, ss)
            self.submitBuildSet(bs)


class LatestFileURL:
    sortByDateString = "?C=M;O=A"

    def _retrievePageAtURL(self):
        content = []
        try:
            opener = urllib.URLopener()
            page = opener.open(self.url + self.sortByDateString)
            content = page.readlines()
            opener.close()
        except:
            print "unable to retrieve page at: " + self.url + " dated: " + self.sortByDateString
        return content
    
    def _populateDict(self):
        '''Extract the latest filename from the given URL
            * retrieve page at URL
            * for each line
                * grab filename URL
                * if filename matches filenameSearchString
                    * grab datetime
                    * store date and filename URL in dictionary
            * return latest filenameURL matching date'''
        
        reDatestamp = re.compile('\d\d-[a-zA-Z]{3,3}-\d\d\d\d \d\d:\d\d')
        reHREF = re.compile('href\="((?:\w|[.-])*)"')
        content = self._retrievePageAtURL()
        for line in content:
            matchHREF = re.search(reHREF, line)
            if matchHREF:
                if self.filenameSearchString in matchHREF.group(1):
                    datetimeMatch = re.search(reDatestamp, line)
                    if datetimeMatch:
                        dts = line[datetimeMatch.start():datetimeMatch.end()]
                        timestamp = datetime(*strptime(dts, '%d-%b-%Y %H:%M')[0:6])
                        self.dateFileDict[timestamp] = matchHREF.group(1)
        
    
    def _getDateKeys(self):
        dbKeys = self.dateFileDict.keys()
        if not dbKeys:
            return []
        return dbKeys
    
    def getLatestFilename(self):
        dateKeys = self._getDateKeys()
        if not dateKeys: return ""
        return self.dateFileDict[max(dateKeys)]
    
    def _printDictionary(self):
        keys = self._getDateKeys()
        for timestamp in keys:
            print "%s : %s" % (timestamp, self.dateFileDict[timestamp])
    
    def testrun(self):
        self._populateDict()
        self._printDictionary()
        name = self.getLatestFilename()
        print self.url + name
    
    def __init__(self, url, filenameSearchString):
        self.url = url
        self.filenameSearchString = filenameSearchString
        self.dateFileDict = {}
        self._populateDict()
    
class MozillaWgetLatestDated(ShellCommand):
    """Download built Firefox client from dated staging directory."""
    haltOnFailure = True
    
    def __init__(self, branch="HEAD", command=None, **kwargs):
        ShellCommand.__init__(self, **kwargs)
        self.addFactoryArguments(branch=branch, command=command)
        self.branch = branch
        self.command = command or ["wget"]

    def setBuild(self, build):
        ShellCommand.setBuild(self, build)
        self.changes = build.source.changes
        #a full url is always provided by sendchange
        self.fileURL = self.changes[-1].files[0]
        self.filename = os.path.basename(self.fileURL)
        # Lie about when we started so the talos runs line up with builds on
        # the waterfall
        try:
            timestamp = int(os.path.basename(os.path.dirname(self.fileURL)))
            self.changes[-1].when = timestamp
        except:
            pass

    def getFilename(self):
        return self.filename
    
    def describe(self, done=False):
        return ["Wget Download"]
    
    def start(self):
        #urlGetter = LatestFileURL(self.url, self.filenameSearchString)
        #self.filename = urlGetter.getLatestFilename()
        #self.fileURL = self.url + self.filename
        if self.branch:
            self.setProperty("fileURL", self.fileURL)
            self.setProperty("filename", self.filename)
        self.setCommand(["wget", "-nv", "-N", self.fileURL])
        ShellCommand.start(self)
    
    def evaluateCommand(self, cmd):
        superResult = ShellCommand.evaluateCommand(self, cmd)
        if SUCCESS != superResult:
            return FAILURE
        if None != re.search('ERROR', cmd.logs['stdio'].getText()):
            return FAILURE
        return SUCCESS

class MozillaInstallZip(ShellCommand):
    """Install given file, unzipping to executablePath"""
    
    def __init__(self, filename="", branch="", command=None, **kwargs):
        ShellCommand.__init__(self, **kwargs)
        self.addFactoryArguments(filename=filename, branch=branch,
                                 command=command)
        self.filename = filename
        self.branch = branch
        self.command = command or ["unzip", "-o"]
    
    def describe(self, done=False):
        return ["Install zip"]
    
    def start(self):
        # removed the mkdir because this happens on the master, not the slave
        if not self.filename:
            if self.branch:
                self.filename = self.getProperty("filename")
            else:
                return FAILURE
        if self.filename:
            self.command = self.command[:] + [self.filename]
        ShellCommand.start(self)
    
    def evaluateCommand(self, cmd):
        superResult = ShellCommand.evaluateCommand(self, cmd)
        if SUCCESS != superResult:
            return FAILURE
        if None != re.search('ERROR', cmd.logs['stdio'].getText()):
            return FAILURE
        if None != re.search('Usage:', cmd.logs['stdio'].getText()):
            return FAILURE
        return SUCCESS

class MozillaWgetSymbols(ShellCommand):
    """Download the matching symbol package with a build."""
    haltOnFailure = False
    _suffixes = ('.tar.bz2', '.dmg', '.zip')

    def __init__(self, **kwargs):
        #def setBuild(self, build):
        #    ShellCommand.setBuild(self, build)
        ShellCommand.__init__(self, **kwargs)

    def describe(self, done=False):
        return ["Wget symbols"]

    def start(self):
        buildURL = self.getProperty('fileURL')

        for suffix in self._suffixes:
            if buildURL.endswith(suffix):
                self.symbolsURL = buildURL[:-len(suffix)] + '.crashreporter-symbols.zip'
                break

        symbolsFile = self.symbolsURL.split('/')[-1]
        self.setProperty('symbolsFile', symbolsFile)
        self.setCommand(['wget', '-nv', '-N', self.symbolsURL])
        ShellCommand.start(self)

class MozillaUnpackSymbols(ShellCommand):
    def __init__(self, **kwargs):
        ShellCommand.__init__(self, **kwargs)

    def describe(self, done=False):
        return ["Unpack symbols"]

    def start(self):
        self.setCommand(['unzip', '-o', '-d', 'symbols', self.getProperty('symbolsFile')])
        ShellCommand.start(self)

class MozillaUpdateConfig(ShellCommand):
    """Configure YAML file for run_tests.py"""
    
    def __init__(self, branch, executablePath, branchName, addOptions=[],
                 useSymbols=False, **kwargs):
        ShellCommand.__init__(self, **kwargs)
        self.addFactoryArguments(branch=branch, executablePath=executablePath,
                                 branchName=branchName, addOptions=addOptions,
                                 useSymbols=useSymbols)
        self.branch = branch
        self.exePath = executablePath
        self.branchName = branchName
        self.addOptions = addOptions
        self.useSymbols = useSymbols

    def setBuild(self, build):
        ShellCommand.setBuild(self, build)
        self.title = build.slavename
        self.changes = build.source.changes
        self.buildid = strftime("%Y%m%d%H%M", localtime(self.changes[-1].when))

        extraopts = copy.copy(self.addOptions)
        if self.useSymbols:
            extraopts += ['--symbolsPath', '../symbols']
        if not self.command:
            self.setCommand(["python", "PerfConfigurator.py", "-v", "-e", self.exePath, "-t", self.title, "-b", self.branch, "-d", self.buildid, '--branchName', self.branchName] + extraopts)

    def describe(self, done=False):
        return ["Update config"]
    
    def evaluateCommand(self, cmd):
        superResult = ShellCommand.evaluateCommand(self, cmd)
        if SUCCESS != superResult:
            return FAILURE
        stdioText = cmd.logs['stdio'].getText()
        if None != re.search('ERROR', stdioText):
            return FAILURE
        if None != re.search('USAGE:', stdioText):
            return FAILURE
        configFileMatch = re.search('outputName\s*=\s*(\w*?.yml)', stdioText)
        if not configFileMatch:
            return FAILURE
        else:
            self.setProperty("configFile", configFileMatch.group(1))
        return SUCCESS
    

class MozillaRunPerfTests(ShellCommand):
    """Run the performance tests"""
    
    def __init__(self, branch, command=None, **kwargs):
        ShellCommand.__init__(self, **kwargs)
        self.addFactoryArguments(branch=branch, command=command)
        self.branch = branch
        self.command = command or ["python", "run_tests.py"]
    
    def describe(self, done=False):
        return ["Run performance tests"]
    
    def createSummary(self, log):
        summary = []
        for line in log.readlines():
            if "RETURN:" in line:
                summary.append(line.replace("RETURN:", "TinderboxPrint:"))
            if "FAIL:" in line:
                summary.append(line.replace("FAIL:", "TinderboxPrint:FAIL:"))
        self.addCompleteLog('summary', "\n".join(summary))
    
    def start(self):
        """docstring for start"""
        self.command = copy.copy(self.command)
        self.command.append(self.getProperty("configFile"))
        ShellCommand.start(self)
    
    def evaluateCommand(self, cmd):
        superResult = ShellCommand.evaluateCommand(self, cmd)
        stdioText = cmd.logs['stdio'].getText()
        if SUCCESS != superResult:
            return FAILURE
        if None != re.search('ERROR', stdioText):
            return FAILURE
        if None != re.search('USAGE:', stdioText):
            return FAILURE
        if None != re.search('FAIL:', stdioText):
            return WARNINGS
        return SUCCESS

class MozillaInstallTarBz2(ShellCommand):
    """Install given file, unzipping to executablePath"""
    
    def __init__(self, filename="", branch="", command=None, **kwargs):
        ShellCommand.__init__(self, **kwargs)
        self.addFactoryArguments(filename=filename, branch=branch,
                                 command=command)
        self.filename = filename
        self.branch = branch
        self.command = command or ["tar", "-jvxf"]
    
    def describe(self, done=False):
        return ["Install tar.bz2"]
    
    def start(self):
        if not self.filename:
            if self.branch:
                self.filename = self.getProperty("filename")
            else:
                return FAILURE
        if self.filename:
            self.command = self.command[:] + [self.filename]
        ShellCommand.start(self)
    
    def evaluateCommand(self, cmd):
        superResult = ShellCommand.evaluateCommand(self, cmd)
        if SUCCESS != superResult:
            return FAILURE
        return SUCCESS

class MozillaInstallTarGz(ShellCommand):
    """Install given file, unzipping to executablePath"""
    
    def __init__(self, filename="", branch="", command=None, **kwargs):
        ShellCommand.__init__(self, **kwargs)
        self.addFactoryArguments(filename=filename, branch=branch,
                                 command=command)
        self.filename = filename
        self.branch = branch
        self.command = command or ["tar", "-zvxf"]
    
    def describe(self, done=False):
        return ["Install tar.gz"]
    
    def start(self):
        if not self.filename:
            if self.branch:
                self.filename = self.getProperty("filename")
            else:
                return FAILURE
        if self.filename:
            self.command = self.command[:] + [self.filename]
        ShellCommand.start(self)
    
    def evaluateCommand(self, cmd):
        superResult = ShellCommand.evaluateCommand(self, cmd)
        if SUCCESS != superResult:
            return FAILURE
        return SUCCESS

class MozillaInstallDmg(ShellCommand):
    """Install given file, copying to workdir"""
    
    def __init__(self, filename="", branch="", command=None, **kwargs):
        ShellCommand.__init__(self, **kwargs)
        self.addFactoryArguments(filename=filename, branch=branch,
                                 command=command)
        self.filename = filename
        self.branch = branch
        self.command = command or ["bash", "installdmg.sh"]
    
    def describe(self, done=False):
        return ["Install dmg"]
    
    def start(self):
        if not self.filename:
            if self.branch:
                self.filename = self.getProperty("filename")
            else:
                return FAILURE
        if self.filename:
            self.command = self.command[:] + [self.filename]
        ShellCommand.start(self)
    
    def evaluateCommand(self, cmd):
        superResult = ShellCommand.evaluateCommand(self, cmd)
        if SUCCESS != superResult:
            return FAILURE
        return SUCCESS

class MozillaInstallDmgEx(ShellCommand):
    """Install given file, copying to workdir"""
    #This is a temporary class to test the new InstallDmg script without affecting the production mac machines
    # if this works everything should be switched over the using it
    
    def __init__(self, filename="", branch="", command=None, **kwargs):
        ShellCommand.__init__(self, **kwargs)
        self.addFactoryArguments(filename=filename, branch=branch,
                                 command=command)
        self.filename = filename
        self.branch = branch
        self.command = command or ["expect", "installdmg.ex"]
    
    def describe(self, done=False):
        return ["Install dmg"]
    
    def start(self):
        if not self.filename:
            if self.branch:
                self.filename = self.getProperty("filename")
            else:
                return FAILURE
        if self.filename:
            self.command = self.command[:] + [self.filename]
        ShellCommand.start(self)
    
    def evaluateCommand(self, cmd):
        superResult = ShellCommand.evaluateCommand(self, cmd)
        if SUCCESS != superResult:
            return FAILURE
        return SUCCESS

class TalosFactory(BuildFactory):

    winClean       = ["touch temp.zip &", "rm", "-rf", "*.zip", "talos/", "firefox/", "symbols/"]
    macClean       = "rm -vrf *"
    linuxClean     = "rm -vrf *"

    def __init__(self, OS, envName, buildBranch, branchName, configOptions,
            buildPath, talosCmd, customManifest='',
            cvsRoot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot",
            workdirBase=None, fetchSymbols=False, plugins='zips/plugins.zip', pagesets='zips/pagesets.zip'):
        BuildFactory.__init__(self)
        if OS in ('linux', 'linuxbranch',):
            cleanCmd = self.linuxClean
        elif OS in ('win',):
            cleanCmd = self.winClean
        else:
            cleanCmd = self.macClean
        if workdirBase is None:
            workdirBase = "."
        self.addStep(ShellCommand(
                           workdir=workdirBase,
                           description="Cleanup",
                           command=cleanCmd,
                           env=MozillaEnvironments[envName]))
#self.addStep(ShellCommand,
#                           command=["cvs", "-d", cvsRoot, "co", "-d", "talos",
#                                    "mozilla/testing/performance/talos"],
#                           workdir=workdirBase,
#                           description="checking out talos",
#                           haltOnFailure=True,
#                           flunkOnFailure=True,
#                           env=MozillaEnvironments[envName])
        self.addStep(FileDownload(
                           mastersrc="talos.zip",
                           slavedest="talos.zip",
                           workdir=workdirBase))
        self.addStep(FileDownload,
                           mastersrc="scripts/count_and_reboot.py",
                           slavedest="count_and_reboot.py",
                           workdir=workdirBase)
        self.addStep(MozillaInstallZip(
                           workdir=workdirBase,
                           branch=buildBranch,
                           haltOnFailure=True,
                           env=MozillaEnvironments[envName],
                           filename="talos.zip"))
        self.addStep(FileDownload(
                           mastersrc="scripts/generate-tpcomponent.py",
                           slavedest="generate-tpcomponent.py",
                           workdir=os.path.join(workdirBase, "talos/page_load_test")))
        if plugins != '':
            self.addStep(FileDownload(
             mastersrc=plugins,
             slavedest=os.path.basename(plugins),
             workdir="talos/base_profile",
             blocksize=640*1024,
            ))
            self.addStep(MozillaInstallZip(
             workdir="talos/base_profile",
             branch=buildBranch,
             haltOnFailure=True,
             env=MozillaEnvironments[envName],
             filename=os.path.basename(plugins)))
                               
        if pagesets != '':      
            self.addStep(FileDownload(
             mastersrc=pagesets,
             slavedest=os.path.basename(pagesets),
             workdir="talos/page_load_test",
             blocksize=640*1024,
            ))
            self.addStep(MozillaInstallZip(
             workdir="talos/page_load_test",
             branch=buildBranch,
             haltOnFailure=True,
             env=MozillaEnvironments[envName],
             filename=os.path.basename(pagesets)))

        if customManifest != '':
            self.addStep(FileDownload(
                           mastersrc=customManifest,
                           slavedest="tp3.manifest",
                           workdir=os.path.join(workdirBase, "talos/page_load_test")))
        self.addStep(ShellCommand(
                           command=["python", "generate-tpcomponent.py"],
                           workdir=os.path.join(workdirBase, "talos/page_load_test"),
                           description="setting up pageloader",
                           haltOnFailure=True,
                           flunkOnFailure=True,
                           env=MozillaEnvironments[envName]))
        self.addStep(MozillaWgetLatestDated(
                           workdir=workdirBase,
                           branch=buildBranch,
                           env=MozillaEnvironments[envName]))
        if fetchSymbols:
            self.addStep(MozillaWgetSymbols(workdir=workdirBase))
            self.addStep(MozillaUnpackSymbols(workdir=workdirBase))

        #install the browser, differs based upon platform
        if OS == 'linux':
            self.addStep(MozillaInstallTarBz2(
                               workdir=workdirBase,
                               branch=buildBranch,
                               haltOnFailure=True,
                               env=MozillaEnvironments[envName]))
        elif OS == 'linuxbranch': #special case for old linux builds
            self.addStep(MozillaInstallTarGz(
                           workdir=workdirBase,
                           branch=buildBranch,
                           haltOnFailure=True,
                           env=MozillaEnvironments[envName]))
        elif OS == 'win':
            self.addStep(MozillaInstallZip(
                               workdir=workdirBase,
                               branch="1.9",
                               haltOnFailure=True,
                               env=MozillaEnvironments[envName]))
            self.addStep(ShellCommand(
                               workdir="firefox/",
                               flunkOnFailure=False,
                               warnOnFailure=False,
                               description="chmod files (see msys bug)",
                               command=["chmod", "-v", "-R", "a+x", "."],
                               env=MozillaEnvironments[envName]))
        elif OS == 'tiger':
            self.addStep(FileDownload(
                           mastersrc="scripts/installdmg.sh",
                           slavedest="installdmg.sh",
                           workdir=workdirBase))
            self.addStep(MozillaInstallDmg(
                               workdir=workdirBase,
                               branch=buildBranch,
                               haltOnFailure=True,
                               env=MozillaEnvironments[envName]))
        else: #leopard
            self.addStep(FileDownload(
                           mastersrc="scripts/installdmg.ex",
                           slavedest="installdmg.ex",
                           workdir=workdirBase))
            self.addStep(MozillaInstallDmgEx(
                               workdir=workdirBase,
                               branch=buildBranch,
                               haltOnFailure=True,
                               env=MozillaEnvironments[envName]))
        self.addStep(MozillaUpdateConfig(
                           workdir=os.path.join(workdirBase, "talos/"),
                           branch=buildBranch,
                           branchName=branchName,
                           haltOnFailure=True,
                           executablePath=buildPath,
                           addOptions=configOptions,
                           env=MozillaEnvironments[envName],
                           useSymbols=fetchSymbols))
        self.addStep(MozillaRunPerfTests(
                           warnOnWarnings=True,
                           workdir=os.path.join(workdirBase, "talos/"),
                           branch=buildBranch,
                           timeout=21600,
                           haltOnFailure=True,
                           command=talosCmd,
                           env=MozillaEnvironments[envName]))
        self.addStep(ShellCommand(
                           flunkOnFailure=False,
                           warnOnFailure=False,
                           alwaysRun=True,
                           workdir=workdirBase,
                           description="reboot after 5 test runs",
                           command=["python", "count_and_reboot.py", "-f", "../talos_count.txt", "-n", "5", "-z"],
                           env=MozillaEnvironments[envName]))

def main(argv=None):
    if argv is None:
        argv = sys.argv
    tester = LatestFileURL('http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-central-linux/1222921630/ ', "en-US.linux-i686.tar.bz2")
    tester.testrun()
    return 0

if __name__ == '__main__':
    main()