author | Sam Garrett <samdgarrett@gmail.com> |
Sat, 27 Oct 2012 17:42:25 -0400 | |
changeset 112131 | e43a2b8ee8400004348ce7efe61c9b6ae150944a |
parent 112130 | 505785fec80e701751bbf453a64357b50320f316 |
child 112132 | 2f8306353ae9c3f20f58272524f4b02d23003743 |
push id | 23798 |
push user | ryanvm@gmail.com |
push date | Sat, 03 Nov 2012 00:06:35 +0000 |
treeherder | mozilla-central@6134edeea902 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | jgriffin, DONTBUILD |
bugs | 771554 |
milestone | 19.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
testing/tps/setup.py | file | annotate | diff | comparison | revisions | |
testing/tps/tps/cli.py | file | annotate | diff | comparison | revisions | |
testing/tps/tps/emailtemplate.py | file | annotate | diff | comparison | revisions | |
testing/tps/tps/sendemail.py | file | annotate | diff | comparison | revisions | |
testing/tps/tps/testrunner.py | file | annotate | diff | comparison | revisions | |
testing/tps/tps/thread.py | file | annotate | diff | comparison | revisions |
--- a/testing/tps/setup.py +++ b/testing/tps/setup.py @@ -1,20 +1,20 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import sys from setuptools import setup, find_packages -version = '0.3' +version = '0.4' deps = ['mozinfo >= 0.3.3', 'mozprofile >= 0.4', 'mozprocess >= 0.4', 'mozrunner >= 5.8', 'mozinstall >= 1.4', - 'mozautolog >= 0.2.4', 'mozautoeslib >= 0.1.1', 'httplib2 >= 0.7.3'] + 'httplib2 >= 0.7.3'] # we only support python 2.6+ right now assert sys.version_info[0] == 2 assert sys.version_info[1] >= 6 setup(name='tps', version=version, description='run automated multi-profile sync tests',
--- a/testing/tps/tps/cli.py +++ b/testing/tps/tps/cli.py @@ -3,46 +3,40 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import json import logging import optparse import os import sys import time -import traceback from threading import RLock from tps import TPSFirefoxRunner, TPSTestRunner def main(): parser = optparse.OptionParser() - parser.add_option("--email-results", - action = "store_true", dest = "emailresults", - default = False, - help = "email the test results to the recipients defined " - "in the config file") parser.add_option("--mobile", action = "store_true", dest = "mobile", default = False, help = "run with mobile settings") - parser.add_option("--autolog", - action = "store_true", dest = "autolog", - default = False, - help = "post results to Autolog") parser.add_option("--testfile", action = "store", type = "string", dest = "testfile", default = '../../services/sync/tests/tps/test_sync.js', help = "path to the test file to run " "[default: %default]") parser.add_option("--logfile", action = "store", type = "string", dest = "logfile", default = 'tps.log', help = "path to the log file [default: %default]") + parser.add_option("--resultfile", + action = "store", type = "string", dest = "resultfile", + default = 'tps_result.json', + help = "path to the result file [default: %default]") parser.add_option("--binary", action = "store", type = "string", dest = "binary", default = None, help = "path to the Firefox binary, specified either as " "a local file or a url; if omitted, the PATH " "will be searched;") parser.add_option("--configfile", action = "store", type = "string", dest = "configfile", @@ -85,50 +79,20 @@ def main(): if sys.platform == 'win32': # replace msys-style paths with proper Windows paths import re m = re.match('^\/\w\/', extensionDir) if m: extensionDir = "%s:/%s" % (m.group(0)[1:2], extensionDir[3:]) extensionDir = extensionDir.replace("/", "\\") - if options.binary is None: - while True: - try: - # If no binary is specified, start the pulse build monitor, and wait - # until we receive build notifications before running tests. - monitor = TPSPulseMonitor(extensionDir, - config=config, - autolog=options.autolog, - emailresults=options.emailresults, - testfile=options.testfile, - logfile=options.logfile, - rlock=rlock) - print "waiting for pulse build notifications" - - if options.pulsefile: - # For testing purposes, inject a pulse message directly into - # the monitor. - builddata = json.loads(open(options.pulsefile, 'r').read()) - monitor.onBuildComplete(builddata) - - monitor.listen() - except KeyboardInterrupt: - sys.exit() - except: - traceback.print_exc() - print 'sleeping 5 minutes' - time.sleep(300) - TPS = TPSTestRunner(extensionDir, - emailresults=options.emailresults, testfile=options.testfile, logfile=options.logfile, binary=options.binary, config=config, rlock=rlock, mobile=options.mobile, - autolog=options.autolog, ignore_unused_engines=options.ignore_unused_engines) TPS.run_tests() if __name__ == "__main__": main()
deleted file mode 100644 --- a/testing/tps/tps/emailtemplate.py +++ /dev/null @@ -1,158 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import datetime - -def GenerateEmailBody(data, numpassed, numfailed, serverUrl, buildUrl): - - now = datetime.datetime.now() - builddate = datetime.datetime.strptime(data['productversion']['buildid'], - '%Y%m%d%H%M%S') - tree = data['productversion']['repository'] - - row = """ -<tr> - <td><a href="http://hg.mozilla.org/services/services-central/file/default/services/sync/tests/tps/{name}">{name}</a></td> - <td>{state}</td> - <td>{message}</td> -</tr> -""" - - rowWithLog = """ -<tr> - <td><a href="http://hg.mozilla.org/services/services-central/file/default/services/sync/tests/tps/{name}">{name}</a></td> - <td>{state}</td> - <td>{message} [<a href="{logurl}">view log</a>]</td> -</tr> -""" - - rows = "" - for test in data['tests']: - if test.get('logurl'): - rows += rowWithLog.format(name=test['name'], - state=test['state'], - message=test['message'] if test['message'] else 'None', - logurl=test['logurl']) - else: - rows += row.format(name=test['name'], - state=test['state'], - message=test['message'] if test['message'] else 'None') - - firefox_version = data['productversion']['version'] - if buildUrl is not None: - firefox_version = "<a href='%s'>%s</a>" % (buildUrl, firefox_version) - body = """ -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> -<head> - <title>TPS</title> - <style type="text/css"> -#headertable {{ border: solid 1px black; margin-bottom: 2em; border-collapse: collapse; font-size: 0.8em; }} -#headertable th {{ border: solid 1px black; background-color: lightgray; padding: 4px; }} -#headertable td {{ border: solid 1px black; padding: 4px; }} -.light {{ color: gray; }} -.pass, a.pass:link, a.pass:visited {{ color: green; font-weight: bold; }} -.fail, a.fail:link, a.fail:visited {{ color: red; font-weight: bold; }} -.rightgray {{ text-align: right; background-color: lightgray; }} -#summarytable {{ border: solid 1px black; margin-bottom: 2em; border-collapse: collapse; font-size: 0.8em; }} -#summarytable th {{ border: solid 1px black; background-color: lightgray; padding: 4px; }} -#summarytable td {{ border: solid 1px black; padding: 4px; }} -</style> -</head> - -<body> - <div id="content"> - -<h2>TPS Testrun Details</h2> - -<table id="headertable"> - -<tr> - <td class="rightgray">Testrun Date</td> - <td>{date}</td> - -</tr> -<tr> - <td class="rightgray">Firefox Version</td> - <td>{firefox_version}</td> -</tr> -<tr> - <td class="rightgray">Firefox Build Date</td> - <td>{firefox_date}</td> -</tr> - -<tr> - <td class="rightgray">Firefox Sync Version / Type</td> - <td>{sync_version} / {sync_type} - </td> -</tr> -<tr> - <td class="rightgray">Firefox Sync Changeset</td> - <td> - - <a href="{repository}/rev/{changeset}"> - - {changeset}</a> / {sync_tree} - - </td> -</tr> -<tr> - <td class="rightgray">Sync Server</td> - <td>{server}</td> -</tr> -<tr> - <td class="rightgray">OS</td> - <td>{os}</td> -</tr> -<tr> - <td class="rightgray">Passed Tests</td> - - <td> - <span class="{passclass}">{numpassed}</span> - </td> -</tr> -<tr> - <td class="rightgray">Failed Tests</td> - <td> - - <span class="{failclass}">{numfailed}</span> - </td> -</tr> -</table> - - -<table id="summarytable"> -<thead> -<tr> - <th>Testcase</th> - <th>Result</th> - <th>Message</th> -</tr> -</thead> - -{rows} - -</table> - - </div> -</body> -</html> - -""".format(date=now.ctime(), - firefox_version=firefox_version, - firefox_date=builddate.ctime(), - sync_version=data['addonversion']['version'], - sync_type=data['synctype'], - sync_tree=tree[tree.rfind("/") + 1:], - repository=data['productversion']['repository'], - changeset=data['productversion']['changeset'], - os=data['os'], - rows=rows, - numpassed=numpassed, - numfailed=numfailed, - passclass="pass" if numpassed > 0 else "light", - failclass="fail" if numfailed > 0 else "light", - server=serverUrl if serverUrl != "" else "default" - ) - - return body
deleted file mode 100644 --- a/testing/tps/tps/sendemail.py +++ /dev/null @@ -1,50 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import smtplib -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -def SendEmail(From=None, To=None, Subject='No Subject', - TextData=None, HtmlData=None, - Server='mail.mozilla.com', Port=465, - Username=None, Password=None): - """Sends an e-mail. - - From is an e-mail address, To is a list of e-mail adresses. - - TextData and HtmlData are both strings. You can specify one or both. - If you specify both, the e-mail will be sent as a MIME multipart - alternative; i.e., the recipient will see the HTML content if his - viewer supports it, otherwise he'll see the text content. - """ - - if From is None or To is None: - raise Exception("Both From and To must be specified") - if TextData is None and HtmlData is None: - raise Exception("Must specify either TextData or HtmlData") - - server = smtplib.SMTP_SSL(Server, Port) - - if Username is not None and Password is not None: - server.login(Username, Password) - - if HtmlData is None: - msg = MIMEText(TextData) - elif TextData is None: - msg = MIMEMultipart() - msg.preamble = Subject - msg.attach(MIMEText(HtmlData, 'html')) - else: - msg = MIMEMultipart('alternative') - msg.attach(MIMEText(TextData, 'plain')) - msg.attach(MIMEText(HtmlData, 'html')) - - msg['Subject'] = Subject - msg['From'] = From - msg['To'] = ', '.join(To) - - server.sendmail(From, To, msg.as_string()) - - server.quit()
--- a/testing/tps/tps/testrunner.py +++ b/testing/tps/tps/testrunner.py @@ -41,17 +41,16 @@ class TempFile(object): def cleanup(self): if self.fd: self.close() if os.access(self.filename, os.F_OK): os.remove(self.filename) __del__ = cleanup - class TPSTestRunner(object): default_env = { 'MOZ_CRASHREPORTER_DISABLE': '1', 'GNOME_DISABLE_CRASH_DIALOG': '1', 'XRE_NO_WINDOWS_CRASH_DIALOG': '1', 'MOZ_NO_REMOTE': '1', 'XPCOM_DEBUG_BREAK': 'warn', } @@ -81,36 +80,36 @@ class TPSTestRunner(object): } syncVerRe = re.compile( r"Sync version: (?P<syncversion>.*)\n") ffVerRe = re.compile( r"Firefox version: (?P<ffver>.*)\n") ffDateRe = re.compile( r"Firefox builddate: (?P<ffdate>.*)\n") - def __init__(self, extensionDir, emailresults=False, testfile="sync.test", + def __init__(self, extensionDir, + testfile="sync.test", binary=None, config=None, rlock=None, mobile=False, - autolog=False, logfile="tps.log", + logfile="tps.log", resultfile="tps_result.json", ignore_unused_engines=False): self.extensions = [] - self.emailresults = emailresults self.testfile = testfile self.logfile = os.path.abspath(logfile) + self.resultfile = resultfile self.binary = binary self.ignore_unused_engines = ignore_unused_engines self.config = config if config else {} self.repo = None self.changeset = None self.branch = None self.numfailed = 0 self.numpassed = 0 self.nightly = False self.rlock = rlock self.mobile = mobile - self.autolog = autolog self.tpsxpi = None self.firefoxRunner = None self.extensionDir = extensionDir self.productversion = None self.addonversion = None self.postdata = {} self.errorlogs = {} @@ -127,16 +126,35 @@ class TPSTestRunner(object): """Appends a string to the logfile""" f = open(self.logfile, 'a') f.write(msg) f.close() if printToConsole: print msg + def writeToResultFile(self, postdata, body=None, + sendTo='crossweave@mozilla.com'): + """Writes results to test file""" + f = open(self.resultfile, 'a') + if body is not None: + postdata['body'] = body + if self.numpassed is not None: + postdata['numpassed'] = self.numpassed + if self.numfailed is not None: + postdata['numfailed'] = self.numfailed + if self.firefoxRunner and self.firefoxRunner.url: + postdata['firefoxrunnerurl'] = self.firefoxRunner.url + + postdata['sendTo'] = sendTo + results = {} + results['results'] = postdata + f.write(json.dumps(results, indent=2)) + f.close() + def _zip_add_file(self, zip, file, rootDir): zip.write(os.path.join(rootDir, file), file) def _zip_add_dir(self, zip, dir, rootDir): try: zip.write(os.path.join(rootDir, dir), dir) except: # on some OS's, adding directory entries doesn't seem to work @@ -309,36 +327,29 @@ class TPSTestRunner(object): # now, run the test group self.run_test_group() except: traceback.print_exc() self.numpassed = 0 self.numfailed = 1 - if self.emailresults: - try: - self.sendEmail('<pre>%s</pre>' % traceback.format_exc(), - sendTo='crossweave@mozilla.com') - except: - traceback.print_exc() - else: - raise - + try: + self.writeToResultFile(self.postdata, + '<pre>%s</pre>' % traceback.format_exc()) + except: + traceback.print_exc() else: try: - if self.autolog: - self.postToAutolog() - if self.emailresults: - self.sendEmail() + self.writeToResultFile(self.postdata) except: traceback.print_exc() try: - self.sendEmail('<pre>%s</pre>' % traceback.format_exc(), - sendTo='crossweave@mozilla.com') + self.writeToResultFile(self.postdata, + '<pre>%s</pre>' % traceback.format_exc()) except: traceback.print_exc() # release our lock if self.rlock: self.rlock.release() # dump out a summary of test results @@ -404,105 +415,8 @@ class TPSTestRunner(object): # generate the postdata we'll use to post the results to the db self.postdata = { 'tests': self.results, 'os':os_string, 'testtype': 'crossweave', 'productversion': self.productversion, 'addonversion': self.addonversion, 'synctype': self.synctype, } - - def sendEmail(self, body=None, sendTo=None): - # send the result e-mail - if self.config.get('email') and self.config['email'].get('username') \ - and self.config['email'].get('password'): - - from tps.sendemail import SendEmail - from tps.emailtemplate import GenerateEmailBody - - if body is None: - buildUrl = None - if self.firefoxRunner and self.firefoxRunner.url: - buildUrl = self.firefoxRunner.url - body = GenerateEmailBody(self.postdata, - self.numpassed, - self.numfailed, - self.config['account']['serverURL'], - buildUrl) - - subj = "TPS Report: " - if self.numfailed == 0 and self.numpassed > 0: - subj += "YEEEAAAHHH" - else: - subj += "PC LOAD LETTER" - - changeset = self.postdata['productversion']['changeset'] if \ - self.postdata and self.postdata.get('productversion') and \ - self.postdata['productversion'].get('changeset') \ - else 'unknown' - subj +=", changeset " + changeset + "; " + str(self.numfailed) + \ - " failed, " + str(self.numpassed) + " passed" - - To = [sendTo] if sendTo else None - if not To: - if self.numfailed > 0 or self.numpassed == 0: - To = self.config['email'].get('notificationlist') - else: - To = self.config['email'].get('passednotificationlist') - - if To: - SendEmail(From=self.config['email']['username'], - To=To, - Subject=subj, - HtmlData=body, - Username=self.config['email']['username'], - Password=self.config['email']['password']) - - def postToAutolog(self): - from mozautolog import RESTfulAutologTestGroup as AutologTestGroup - - group = AutologTestGroup( - harness='crossweave', - testgroup='crossweave-%s' % self.synctype, - server=self.config.get('es'), - restserver=self.config.get('restserver'), - machine=socket.gethostname(), - platform=self.config.get('platform', None), - os=self.config.get('os', None), - ) - tree = self.postdata['productversion']['repository'] - group.set_primary_product( - tree=tree[tree.rfind("/")+1:], - version=self.postdata['productversion']['version'], - buildid=self.postdata['productversion']['buildid'], - buildtype='opt', - revision=self.postdata['productversion']['changeset'], - ) - group.add_test_suite( - passed=self.numpassed, - failed=self.numfailed, - todo=0, - ) - for test in self.results: - if test['state'] != "TEST-PASS": - errorlog = self.errorlogs.get(test['name']) - errorlog_filename = errorlog.filename if errorlog else None - group.add_test_failure( - test = test['name'], - status = test['state'], - text = test['message'], - logfile = errorlog_filename - ) - try: - group.submit() - except: - self.sendEmail('<pre>%s</pre>' % traceback.format_exc(), - sendTo='crossweave@mozilla.com') - return - - # Iterate through all testfailure objects, and update the postdata - # dict with the testfailure logurl's, if any. - for tf in group.testsuites[-1].testfailures: - result = [x for x in self.results if x.get('name') == tf.test] - if not result: - continue - result[0]['logurl'] = tf.logurl -
--- a/testing/tps/tps/thread.py +++ b/testing/tps/tps/thread.py @@ -3,71 +3,62 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from threading import Thread from testrunner import TPSTestRunner class TPSTestThread(Thread): - def __init__(self, extensionDir, builddata=None, emailresults=False, - testfile=None, logfile=None, rlock=None, config=None, - autolog=False): + def __init__(self, extensionDir, builddata=None, + testfile=None, logfile=None, rlock=None, config=None): assert(builddata) assert(config) self.extensionDir = extensionDir self.builddata = builddata - self.emailresults = emailresults self.testfile = testfile self.logfile = logfile self.rlock = rlock self.config = config - self.autolog = autolog Thread.__init__(self) def run(self): # run the tests in normal mode ... TPS = TPSTestRunner(self.extensionDir, - emailresults=self.emailresults, testfile=self.testfile, logfile=self.logfile, binary=self.builddata['buildurl'], config=self.config, rlock=self.rlock, - mobile=False, - autolog=self.autolog) + mobile=False) TPS.run_tests() # Get the binary used by this TPS instance, and use it in subsequent # ones, so it doesn't have to be re-downloaded each time. binary = TPS.firefoxRunner.binary # ... and then again in mobile mode TPS_mobile = TPSTestRunner(self.extensionDir, - emailresults=self.emailresults, testfile=self.testfile, logfile=self.logfile, binary=binary, config=self.config, rlock=self.rlock, - mobile=True, - autolog=self.autolog) + mobile=True) TPS_mobile.run_tests() # ... and again via the staging server, if credentials are present stageaccount = self.config.get('stageaccount') if stageaccount: username = stageaccount.get('username') password = stageaccount.get('password') passphrase = stageaccount.get('passphrase') if username and password and passphrase: stageconfig = self.config.copy() stageconfig['account'] = stageaccount.copy() TPS_stage = TPSTestRunner(self.extensionDir, - emailresults=self.emailresults, testfile=self.testfile, logfile=self.logfile, binary=binary, config=stageconfig, rlock=self.rlock, - mobile=False, - autolog=self.autolog) + mobile=False)#, autolog=self.autolog) TPS_stage.run_tests()