Bug 771554 - Removing autolog and having tps write-out results to JSON so that coversheet can handle them, r=jgriffin, DONTBUILD(NPOTB)
authorSam Garrett <samdgarrett@gmail.com>
Sat, 27 Oct 2012 17:42:25 -0400
changeset 112131 e43a2b8ee8400004348ce7efe61c9b6ae150944a
parent 112130 505785fec80e701751bbf453a64357b50320f316
child 112132 2f8306353ae9c3f20f58272524f4b02d23003743
push id23798
push userryanvm@gmail.com
push dateSat, 03 Nov 2012 00:06:35 +0000
treeherdermozilla-central@6134edeea902 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgriffin, DONTBUILD
bugs771554
milestone19.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 771554 - Removing autolog and having tps write-out results to JSON so that coversheet can handle them, r=jgriffin, DONTBUILD(NPOTB)
testing/tps/setup.py
testing/tps/tps/cli.py
testing/tps/tps/emailtemplate.py
testing/tps/tps/sendemail.py
testing/tps/tps/testrunner.py
testing/tps/tps/thread.py
--- a/testing/tps/setup.py
+++ b/testing/tps/setup.py
@@ -1,20 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 from setuptools import setup, find_packages
 
-version = '0.3'
+version = '0.4'
 
 deps = ['mozinfo >= 0.3.3', 'mozprofile >= 0.4',
         'mozprocess >= 0.4', 'mozrunner >= 5.8', 'mozinstall >= 1.4',
-        'mozautolog >= 0.2.4', 'mozautoeslib >= 0.1.1', 'httplib2 >= 0.7.3']
+        'httplib2 >= 0.7.3']
 
 # we only support python 2.6+ right now
 assert sys.version_info[0] == 2
 assert sys.version_info[1] >= 6
 
 setup(name='tps',
       version=version,
       description='run automated multi-profile sync tests',
--- a/testing/tps/tps/cli.py
+++ b/testing/tps/tps/cli.py
@@ -3,46 +3,40 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import logging
 import optparse
 import os
 import sys
 import time
-import traceback
 
 from threading import RLock
 
 from tps import TPSFirefoxRunner, TPSTestRunner
 
 def main():
   parser = optparse.OptionParser()
-  parser.add_option("--email-results",
-                    action = "store_true", dest = "emailresults",
-                    default = False,
-                    help = "email the test results to the recipients defined "
-                           "in the config file")
   parser.add_option("--mobile",
                     action = "store_true", dest = "mobile",
                     default = False,
                     help = "run with mobile settings")
-  parser.add_option("--autolog",
-                    action = "store_true", dest = "autolog",
-                    default = False,
-                    help = "post results to Autolog")
   parser.add_option("--testfile",
                     action = "store", type = "string", dest = "testfile",
                     default = '../../services/sync/tests/tps/test_sync.js',
                     help = "path to the test file to run "
                            "[default: %default]")
   parser.add_option("--logfile",
                     action = "store", type = "string", dest = "logfile",
                     default = 'tps.log',
                     help = "path to the log file [default: %default]")
+  parser.add_option("--resultfile",
+                    action = "store", type = "string", dest = "resultfile",
+                    default = 'tps_result.json',
+                    help = "path to the result file [default: %default]")
   parser.add_option("--binary",
                     action = "store", type = "string", dest = "binary",
                     default = None,
                     help = "path to the Firefox binary, specified either as "
                            "a local file or a url; if omitted, the PATH "
                            "will be searched;")
   parser.add_option("--configfile",
                     action = "store", type = "string", dest = "configfile",
@@ -85,50 +79,20 @@ def main():
     if sys.platform == 'win32':
       # replace msys-style paths with proper Windows paths
       import re
       m = re.match('^\/\w\/', extensionDir)
       if m:
         extensionDir = "%s:/%s" % (m.group(0)[1:2], extensionDir[3:])
         extensionDir = extensionDir.replace("/", "\\")
 
-  if options.binary is None:
-    while True:
-      try:
-        # If no binary is specified, start the pulse build monitor, and wait
-        # until we receive build notifications before running tests.
-        monitor = TPSPulseMonitor(extensionDir,
-                                  config=config,
-                                  autolog=options.autolog,
-                                  emailresults=options.emailresults,
-                                  testfile=options.testfile,
-                                  logfile=options.logfile,
-                                  rlock=rlock)
-        print "waiting for pulse build notifications"
-
-        if options.pulsefile:
-          # For testing purposes, inject a pulse message directly into
-          # the monitor.
-          builddata = json.loads(open(options.pulsefile, 'r').read())
-          monitor.onBuildComplete(builddata)
-
-        monitor.listen()
-      except KeyboardInterrupt:
-        sys.exit()
-      except:
-        traceback.print_exc()
-        print 'sleeping 5 minutes'
-        time.sleep(300)
-
   TPS = TPSTestRunner(extensionDir,
-                      emailresults=options.emailresults,
                       testfile=options.testfile,
                       logfile=options.logfile,
                       binary=options.binary,
                       config=config,
                       rlock=rlock,
                       mobile=options.mobile,
-                      autolog=options.autolog,
                       ignore_unused_engines=options.ignore_unused_engines)
   TPS.run_tests()
 
 if __name__ == "__main__":
   main()
deleted file mode 100644
--- a/testing/tps/tps/emailtemplate.py
+++ /dev/null
@@ -1,158 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import datetime
-
-def GenerateEmailBody(data, numpassed, numfailed, serverUrl, buildUrl):
-
-  now = datetime.datetime.now()
-  builddate = datetime.datetime.strptime(data['productversion']['buildid'],
-                                         '%Y%m%d%H%M%S')
-  tree = data['productversion']['repository']
-
-  row = """
-<tr>
-  <td><a href="http://hg.mozilla.org/services/services-central/file/default/services/sync/tests/tps/{name}">{name}</a></td>
-  <td>{state}</td>
-  <td>{message}</td>
-</tr>
-"""
-
-  rowWithLog = """
-<tr>
-  <td><a href="http://hg.mozilla.org/services/services-central/file/default/services/sync/tests/tps/{name}">{name}</a></td>
-  <td>{state}</td>
-  <td>{message} [<a href="{logurl}">view log</a>]</td>
-</tr>
-"""
-
-  rows = ""
-  for test in data['tests']:
-    if test.get('logurl'):
-      rows += rowWithLog.format(name=test['name'],
-                         state=test['state'],
-                         message=test['message'] if test['message'] else 'None',
-                         logurl=test['logurl'])
-    else:
-      rows += row.format(name=test['name'],
-                         state=test['state'],
-                         message=test['message'] if test['message'] else 'None')
-
-  firefox_version = data['productversion']['version']
-  if buildUrl is not None:
-    firefox_version = "<a href='%s'>%s</a>" % (buildUrl, firefox_version)
-  body = """
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-    <title>TPS</title>
-    <style type="text/css">
-#headertable {{ border: solid 1px black; margin-bottom: 2em; border-collapse: collapse; font-size: 0.8em; }}
-#headertable th {{ border: solid 1px black; background-color: lightgray; padding: 4px; }}
-#headertable td {{ border: solid 1px black; padding: 4px; }}
-.light {{ color: gray; }}
-.pass, a.pass:link, a.pass:visited {{ color: green; font-weight: bold; }}
-.fail, a.fail:link, a.fail:visited {{ color: red; font-weight: bold; }}
-.rightgray {{ text-align: right; background-color: lightgray; }}
-#summarytable {{ border: solid 1px black; margin-bottom: 2em; border-collapse: collapse; font-size: 0.8em; }}
-#summarytable th {{ border: solid 1px black; background-color: lightgray; padding: 4px; }}
-#summarytable td {{ border: solid 1px black; padding: 4px; }}
-</style>
-</head>
-
-<body>
-    <div id="content">
-        
-<h2>TPS Testrun Details</h2>
-
-<table id="headertable">
-
-<tr>
-  <td class="rightgray">Testrun Date</td>
-  <td>{date}</td>
-
-</tr>
-<tr>
-  <td class="rightgray">Firefox Version</td>
-  <td>{firefox_version}</td>
-</tr>
-<tr>
-  <td class="rightgray">Firefox Build Date</td>
-  <td>{firefox_date}</td>
-</tr>
-
-<tr>
-  <td class="rightgray">Firefox Sync Version / Type</td>
-  <td>{sync_version} / {sync_type}
-  </td>
-</tr>
-<tr>
-  <td class="rightgray">Firefox Sync Changeset</td>
-  <td>
-
-    <a href="{repository}/rev/{changeset}">
-
-      {changeset}</a> / {sync_tree}
-
-  </td>
-</tr>
-<tr>
-  <td class="rightgray">Sync Server</td>
-  <td>{server}</td>
-</tr>
-<tr>
-  <td class="rightgray">OS</td>
-  <td>{os}</td>
-</tr>
-<tr>
-  <td class="rightgray">Passed Tests</td>
-
-  <td>
-  <span class="{passclass}">{numpassed}</span>
-  </td>
-</tr>
-<tr>
-  <td class="rightgray">Failed Tests</td>
-  <td>
-
-  <span class="{failclass}">{numfailed}</span>
-  </td>
-</tr>
-</table>
-
-
-<table id="summarytable">
-<thead>
-<tr>
-  <th>Testcase</th>
-  <th>Result</th>
-  <th>Message</th>
-</tr>
-</thead>
-
-{rows}
-
-</table>
-
-    </div>
-</body>
-</html>
-
-""".format(date=now.ctime(),
-           firefox_version=firefox_version,
-           firefox_date=builddate.ctime(),
-           sync_version=data['addonversion']['version'],
-           sync_type=data['synctype'],
-           sync_tree=tree[tree.rfind("/") + 1:],
-           repository=data['productversion']['repository'],
-           changeset=data['productversion']['changeset'],
-           os=data['os'],
-           rows=rows,
-           numpassed=numpassed,
-           numfailed=numfailed,
-           passclass="pass" if numpassed > 0 else "light",
-           failclass="fail" if numfailed > 0 else "light",
-           server=serverUrl if serverUrl != "" else "default"
-          )
-
-  return body
deleted file mode 100644
--- a/testing/tps/tps/sendemail.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import smtplib
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
-
-def SendEmail(From=None, To=None, Subject='No Subject', 
-              TextData=None, HtmlData=None,
-              Server='mail.mozilla.com', Port=465,
-              Username=None, Password=None):
-  """Sends an e-mail.
-  
-     From is an e-mail address, To is a list of e-mail adresses.
-     
-     TextData and HtmlData are both strings.  You can specify one or both.
-     If you specify both, the e-mail will be sent as a MIME multipart
-     alternative; i.e., the recipient will see the HTML content if his
-     viewer supports it, otherwise he'll see the text content.
-  """
-
-  if From is None or To is None:
-    raise Exception("Both From and To must be specified")
-  if TextData is None and HtmlData is None:
-    raise Exception("Must specify either TextData or HtmlData")
-
-  server = smtplib.SMTP_SSL(Server, Port)
-
-  if Username is not None and Password is not None:
-    server.login(Username, Password)
-
-  if HtmlData is None:
-    msg = MIMEText(TextData)
-  elif TextData is None:
-    msg = MIMEMultipart()
-    msg.preamble = Subject
-    msg.attach(MIMEText(HtmlData, 'html'))
-  else:
-    msg = MIMEMultipart('alternative')
-    msg.attach(MIMEText(TextData, 'plain'))
-    msg.attach(MIMEText(HtmlData, 'html'))
-
-  msg['Subject'] = Subject
-  msg['From'] = From
-  msg['To'] = ', '.join(To)
-
-  server.sendmail(From, To, msg.as_string())
-
-  server.quit()
--- a/testing/tps/tps/testrunner.py
+++ b/testing/tps/tps/testrunner.py
@@ -41,17 +41,16 @@ class TempFile(object):
   def cleanup(self):
     if self.fd:
       self.close()
     if os.access(self.filename, os.F_OK):
       os.remove(self.filename)
 
   __del__ = cleanup
 
-
 class TPSTestRunner(object):
 
   default_env = { 'MOZ_CRASHREPORTER_DISABLE': '1',
                   'GNOME_DISABLE_CRASH_DIALOG': '1',
                   'XRE_NO_WINDOWS_CRASH_DIALOG': '1',
                   'MOZ_NO_REMOTE': '1',
                   'XPCOM_DEBUG_BREAK': 'warn',
                 }
@@ -81,36 +80,36 @@ class TPSTestRunner(object):
                         }
   syncVerRe = re.compile(
       r"Sync version: (?P<syncversion>.*)\n")
   ffVerRe = re.compile(
       r"Firefox version: (?P<ffver>.*)\n")
   ffDateRe = re.compile(
       r"Firefox builddate: (?P<ffdate>.*)\n")
 
-  def __init__(self, extensionDir, emailresults=False, testfile="sync.test",
+  def __init__(self, extensionDir,
+               testfile="sync.test",
                binary=None, config=None, rlock=None, mobile=False,
-               autolog=False, logfile="tps.log",
+               logfile="tps.log", resultfile="tps_result.json",
                ignore_unused_engines=False):
     self.extensions = []
-    self.emailresults = emailresults
     self.testfile = testfile
     self.logfile = os.path.abspath(logfile)
+    self.resultfile = resultfile
     self.binary = binary
     self.ignore_unused_engines = ignore_unused_engines
     self.config = config if config else {}
     self.repo = None
     self.changeset = None
     self.branch = None
     self.numfailed = 0
     self.numpassed = 0
     self.nightly = False
     self.rlock = rlock
     self.mobile = mobile
-    self.autolog = autolog
     self.tpsxpi = None
     self.firefoxRunner = None
     self.extensionDir = extensionDir
     self.productversion = None
     self.addonversion = None
     self.postdata = {}
     self.errorlogs = {}
 
@@ -127,16 +126,35 @@ class TPSTestRunner(object):
     """Appends a string to the logfile"""
 
     f = open(self.logfile, 'a')
     f.write(msg)
     f.close()
     if printToConsole:
       print msg
 
+  def writeToResultFile(self, postdata, body=None,
+                        sendTo='crossweave@mozilla.com'):
+    """Writes results to test file"""
+    f = open(self.resultfile, 'a')
+    if body is not None:
+      postdata['body'] = body
+    if self.numpassed is not None:
+      postdata['numpassed'] = self.numpassed
+    if self.numfailed is not None:
+      postdata['numfailed'] = self.numfailed
+    if self.firefoxRunner and self.firefoxRunner.url:
+      postdata['firefoxrunnerurl'] = self.firefoxRunner.url
+
+    postdata['sendTo'] = sendTo
+    results = {}
+    results['results'] = postdata
+    f.write(json.dumps(results, indent=2))
+    f.close()
+
   def _zip_add_file(self, zip, file, rootDir):
     zip.write(os.path.join(rootDir, file), file)
 
   def _zip_add_dir(self, zip, dir, rootDir):
     try:
       zip.write(os.path.join(rootDir, dir), dir)
     except:
       # on some OS's, adding directory entries doesn't seem to work
@@ -309,36 +327,29 @@ class TPSTestRunner(object):
 
       # now, run the test group
       self.run_test_group()
 
     except:
       traceback.print_exc()
       self.numpassed = 0
       self.numfailed = 1
-      if self.emailresults:
-        try:
-          self.sendEmail('<pre>%s</pre>' % traceback.format_exc(),
-                         sendTo='crossweave@mozilla.com')
-        except:
-          traceback.print_exc()
-      else:
-        raise
-
+      try:
+        self.writeToResultFile(self.postdata,
+                               '<pre>%s</pre>' % traceback.format_exc())
+      except:
+        traceback.print_exc()
     else:
       try:
-        if self.autolog:
-          self.postToAutolog()
-        if self.emailresults:
-          self.sendEmail()
+        self.writeToResultFile(self.postdata)
       except:
         traceback.print_exc()
         try:
-          self.sendEmail('<pre>%s</pre>' % traceback.format_exc(),
-                         sendTo='crossweave@mozilla.com')
+          self.writeToResultFile(self.postdata,
+                                 '<pre>%s</pre>' % traceback.format_exc())
         except:
           traceback.print_exc()
 
     # release our lock
     if self.rlock:
       self.rlock.release()
 
     # dump out a summary of test results
@@ -404,105 +415,8 @@ class TPSTestRunner(object):
     # generate the postdata we'll use to post the results to the db
     self.postdata = { 'tests': self.results, 
                       'os':os_string,
                       'testtype': 'crossweave',
                       'productversion': self.productversion,
                       'addonversion': self.addonversion,
                       'synctype': self.synctype,
                     }
-
-  def sendEmail(self, body=None, sendTo=None):
-    # send the result e-mail
-    if self.config.get('email') and self.config['email'].get('username') \
-       and self.config['email'].get('password'):
-
-      from tps.sendemail import SendEmail
-      from tps.emailtemplate import GenerateEmailBody
-
-      if body is None:
-        buildUrl = None
-        if self.firefoxRunner and self.firefoxRunner.url:
-          buildUrl = self.firefoxRunner.url
-        body = GenerateEmailBody(self.postdata,
-                                 self.numpassed,
-                                 self.numfailed,
-                                 self.config['account']['serverURL'],
-                                 buildUrl)
-
-      subj = "TPS Report: "
-      if self.numfailed == 0 and self.numpassed > 0:
-        subj += "YEEEAAAHHH"
-      else:
-        subj += "PC LOAD LETTER"
-
-      changeset = self.postdata['productversion']['changeset'] if \
-          self.postdata and self.postdata.get('productversion') and \
-          self.postdata['productversion'].get('changeset') \
-          else 'unknown'
-      subj +=", changeset " + changeset + "; " + str(self.numfailed) + \
-             " failed, " + str(self.numpassed) + " passed"
-
-      To = [sendTo] if sendTo else None
-      if not To:
-        if self.numfailed > 0 or self.numpassed == 0:
-          To = self.config['email'].get('notificationlist')
-        else:
-          To = self.config['email'].get('passednotificationlist')
-
-      if To:
-        SendEmail(From=self.config['email']['username'],
-                  To=To,
-                  Subject=subj,
-                  HtmlData=body,
-                  Username=self.config['email']['username'],
-                  Password=self.config['email']['password'])
-
-  def postToAutolog(self):
-    from mozautolog import RESTfulAutologTestGroup as AutologTestGroup
-
-    group = AutologTestGroup(
-              harness='crossweave',
-              testgroup='crossweave-%s' % self.synctype,
-              server=self.config.get('es'),
-              restserver=self.config.get('restserver'),
-              machine=socket.gethostname(),
-              platform=self.config.get('platform', None),
-              os=self.config.get('os', None),
-            )
-    tree = self.postdata['productversion']['repository']
-    group.set_primary_product(
-              tree=tree[tree.rfind("/")+1:],
-              version=self.postdata['productversion']['version'],
-              buildid=self.postdata['productversion']['buildid'],
-              buildtype='opt',
-              revision=self.postdata['productversion']['changeset'],
-            )
-    group.add_test_suite(
-              passed=self.numpassed,
-              failed=self.numfailed,
-              todo=0,
-            )
-    for test in self.results:
-      if test['state'] != "TEST-PASS":
-        errorlog = self.errorlogs.get(test['name'])
-        errorlog_filename = errorlog.filename if errorlog else None
-        group.add_test_failure(
-              test = test['name'],
-              status = test['state'],
-              text = test['message'],
-              logfile = errorlog_filename
-            )
-    try:
-        group.submit()
-    except:
-        self.sendEmail('<pre>%s</pre>' % traceback.format_exc(),
-                       sendTo='crossweave@mozilla.com')
-        return
-
-    # Iterate through all testfailure objects, and update the postdata
-    # dict with the testfailure logurl's, if any.
-    for tf in group.testsuites[-1].testfailures:
-      result = [x for x in self.results if x.get('name') == tf.test]
-      if not result:
-        continue
-      result[0]['logurl'] = tf.logurl
-
--- a/testing/tps/tps/thread.py
+++ b/testing/tps/tps/thread.py
@@ -3,71 +3,62 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from threading import Thread
 
 from testrunner import TPSTestRunner
 
 class TPSTestThread(Thread):
 
-  def __init__(self, extensionDir, builddata=None, emailresults=False,
-               testfile=None, logfile=None, rlock=None, config=None,
-               autolog=False):
+  def __init__(self, extensionDir, builddata=None,
+               testfile=None, logfile=None, rlock=None, config=None):
     assert(builddata)
     assert(config)
     self.extensionDir = extensionDir
     self.builddata = builddata
-    self.emailresults = emailresults
     self.testfile = testfile
     self.logfile = logfile
     self.rlock = rlock
     self.config = config
-    self.autolog = autolog
     Thread.__init__(self)
 
   def run(self):
     # run the tests in normal mode ...
     TPS = TPSTestRunner(self.extensionDir,
-                        emailresults=self.emailresults,
                         testfile=self.testfile,
                         logfile=self.logfile,
                         binary=self.builddata['buildurl'],
                         config=self.config,
                         rlock=self.rlock,
-                        mobile=False,
-                        autolog=self.autolog)
+                        mobile=False)
     TPS.run_tests()
 
     # Get the binary used by this TPS instance, and use it in subsequent
     # ones, so it doesn't have to be re-downloaded each time.
     binary = TPS.firefoxRunner.binary
 
     # ... and then again in mobile mode
     TPS_mobile = TPSTestRunner(self.extensionDir,
-                               emailresults=self.emailresults,
                                testfile=self.testfile,
                                logfile=self.logfile,
                                binary=binary,
                                config=self.config,
                                rlock=self.rlock,
-                               mobile=True,
-                               autolog=self.autolog)
+                               mobile=True)
     TPS_mobile.run_tests()
 
     # ... and again via the staging server, if credentials are present
     stageaccount = self.config.get('stageaccount')
     if stageaccount:
       username = stageaccount.get('username')
       password = stageaccount.get('password')
       passphrase = stageaccount.get('passphrase')
       if username and password and passphrase:
         stageconfig = self.config.copy()
         stageconfig['account'] = stageaccount.copy()
         TPS_stage = TPSTestRunner(self.extensionDir,
-                                  emailresults=self.emailresults,
                                   testfile=self.testfile,
                                   logfile=self.logfile,
                                   binary=binary,
                                   config=stageconfig,
                                   rlock=self.rlock,
-                                  mobile=False,
-                                  autolog=self.autolog)
+                                  mobile=False)#, autolog=self.autolog)
         TPS_stage.run_tests()