Bug 728294 - Part 1 - Remove old debug log parser; r=ted
authorTim Taubert <tim.taubert@gmx.de>
Fri, 03 Aug 2012 12:36:59 +0200
changeset 106786 c10c3ac392d5f69e3df42cc61b57f5f8b031c11c
parent 106785 98cdb3e2d30acb5dd0e21e2bd0127308132540a9
child 106787 80367942e54c11a62a010d6ec57448c9b087bb58
push id1490
push userakeybl@mozilla.com
push dateMon, 08 Oct 2012 18:29:50 +0000
treeherdermozilla-beta@f335e7dacdc1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs728294
milestone17.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 728294 - Part 1 - Remove old debug log parser; r=ted
build/automation.py.in
build/automationutils.py
build/mobile/remoteautomation.py
testing/mochitest/runtests.py
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -814,17 +814,17 @@ user_pref("camino.use_system_proxy_setti
         # We should have a "crashinject" program in our utility path
         crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))
         if os.path.exists(crashinject) and subprocess.Popen([crashinject, str(proc.pid)]).wait() == 0:
           return
       #TODO: kill the process such that it triggers Breakpad on OS X (bug 525296)
     self.log.info("Can't trigger Breakpad, just killing process")
     proc.kill()
 
-  def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath, logger):
+  def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath):
     """ Look for timeout or crashes and return the status after the process terminates """
     stackFixerProcess = None
     stackFixerFunction = None
     didTimeout = False
     hitMaxTime = False
     if proc.stdout is None:
       self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection")
     else:
@@ -851,18 +851,16 @@ user_pref("camino.use_system_proxy_setti
                                          stdout=subprocess.PIPE)
         logsource = stackFixerProcess.stdout
 
       (line, didTimeout) = self.readWithTimeout(logsource, timeout)
       while line != "" and not didTimeout:
         if stackFixerFunction:
           line = stackFixerFunction(line)
         self.log.info(line.rstrip().decode("UTF-8", "ignore"))
-        if logger:
-          logger.log(line)
         if "TEST-START" in line and "|" in line:
           self.lastTestSeen = line.split("|")[1].strip()
         if not debuggerInfo and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
           if self.haveDumpedScreen:
             self.log.info("Not taking screenshot here: see the one that was previously logged")
           else:
             self.dumpScreen(utilityPath)
 
@@ -942,17 +940,17 @@ user_pref("camino.use_system_proxy_setti
           self.log.info("TEST-UNEXPECTED-FAIL | automation.py | child process %d still alive after shutdown", processPID)
           self.killPid(processPID)
 
   def checkForCrashes(self, profileDir, symbolsPath):
     automationutils.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath, self.lastTestSeen)
 
   def runApp(self, testURL, env, app, profileDir, extraArgs,
              runSSLTunnel = False, utilityPath = None,
-             xrePath = None, certPath = None, logger = None,
+             xrePath = None, certPath = None,
              debuggerInfo = None, symbolsPath = None,
              timeout = -1, maxTime = None):
     """
     Run the app, log the duration it took to execute, return the status code.
     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
     """
 
     if utilityPath == None:
@@ -1001,17 +999,17 @@ user_pref("camino.use_system_proxy_setti
     self.lastTestSeen = "automation.py"
     proc = self.Process([cmd] + args,
                  env = self.environment(env, xrePath = xrePath,
                                    crashreporter = not debuggerInfo),
                  stdout = outputPipe,
                  stderr = subprocess.STDOUT)
     self.log.info("INFO | automation.py | Application pid: %d", proc.pid)
 
-    status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath, logger)
+    status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath)
     self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime))
 
     # Do a final check for zombie child processes.
     self.checkForZombies(processLog)
     self.checkForCrashes(profileDir, symbolsPath)
 
     if os.path.exists(processLog):
       os.unlink(processLog)
--- a/build/automationutils.py
+++ b/build/automationutils.py
@@ -2,30 +2,28 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import with_statement
 import glob, logging, os, platform, shutil, subprocess, sys, tempfile, urllib2, zipfile
 import re
 from urlparse import urlparse
-from operator import itemgetter
 
 __all__ = [
   "ZipFileReader",
   "addCommonOptions",
   "checkForCrashes",
   "dumpLeakLog",
   "isURL",
   "processLeakLog",
   "getDebuggerInfo",
   "DEBUGGER_INFO",
   "replaceBackSlashes",
   "wrapCommand",
-  "ShutdownLeakLogger"
   ]
 
 # Map of debugging programs to information about them, like default arguments
 # and whether or not they are interactive.
 DEBUGGER_INFO = {
   # gdb requires that you supply the '--args' flag in order to pass arguments
   # after the executable name to the executable.
   "gdb": {
@@ -414,120 +412,8 @@ def wrapCommand(cmd):
   binary.
   """
   if platform.system() == "Darwin" and \
      hasattr(platform, 'mac_ver') and \
      platform.mac_ver()[0][:4] < '10.6':
     return ["arch", "-arch", "i386"] + cmd
   # otherwise just execute the command normally
   return cmd
-
-class ShutdownLeakLogger(object):
-  """
-  Parses the mochitest run log when running a debug build, assigns all leaked
-  DOM windows (that are still around after test suite shutdown, despite running
-  the GC) to the tests that created them and prints leak statistics.
-  """
-  MAX_LEAK_COUNT = 4
-
-  def __init__(self, logger):
-    self.logger = logger
-    self.tests = []
-    self.leakedWindows = {}
-    self.leakedDocShells = set()
-    self.currentTest = None
-    self.seenShutdown = False
-
-  def log(self, line):
-    if line[2:11] == "DOMWINDOW":
-      self._logWindow(line)
-    elif line[2:10] == "DOCSHELL":
-      self._logDocShell(line)
-    elif line.startswith("TEST-START"):
-      fileName = line.split(" ")[-1].strip().replace("chrome://mochitests/content/browser/", "")
-      self.currentTest = {"fileName": fileName, "windows": set(), "docShells": set()}
-    elif line.startswith("INFO TEST-END"):
-      # don't track a test if no windows or docShells leaked
-      if self.currentTest and (self.currentTest["windows"] or self.currentTest["docShells"]):
-        self.tests.append(self.currentTest)
-      self.currentTest = None
-    elif line.startswith("INFO TEST-START | Shutdown"):
-      self.seenShutdown = True
-
-  def parse(self):
-    leakingTests = self._parseLeakingTests()
-
-    if leakingTests:
-      totalWindows = sum(len(test["leakedWindows"]) for test in leakingTests)
-      totalDocShells = sum(len(test["leakedDocShells"]) for test in leakingTests)
-      msgType = "TEST-INFO" if totalWindows + totalDocShells <= self.MAX_LEAK_COUNT else "TEST-UNEXPECTED-FAIL"
-      self.logger.info("%s | ShutdownLeaks | leaked %d DOMWindow(s) and %d DocShell(s) until shutdown", msgType, totalWindows, totalDocShells)
-
-    for test in leakingTests:
-      for url, count in self._zipLeakedWindows(test["leakedWindows"]):
-        self.logger.info("%s | %s | leaked %d window(s) until shutdown [url = %s]", msgType, test["fileName"], count, url)
-
-      if test["leakedDocShells"]:
-        self.logger.info("%s | %s | leaked %d docShell(s) until shutdown", msgType, test["fileName"], len(test["leakedDocShells"]))
-
-  def _logWindow(self, line):
-    created = line[:2] == "++"
-    id = self._parseValue(line, "serial")
-
-    # log line has invalid format
-    if not id:
-      return
-
-    if self.currentTest:
-      windows = self.currentTest["windows"]
-      if created:
-        windows.add(id)
-      else:
-        windows.discard(id)
-    elif self.seenShutdown and not created:
-      self.leakedWindows[id] = self._parseValue(line, "url")
-
-  def _logDocShell(self, line):
-    created = line[:2] == "++"
-    id = self._parseValue(line, "id")
-
-    # log line has invalid format
-    if not id:
-      return
-
-    if self.currentTest:
-      docShells = self.currentTest["docShells"]
-      if created:
-        docShells.add(id)
-      else:
-        docShells.discard(id)
-    elif self.seenShutdown and not created:
-      self.leakedDocShells.add(id)
-
-  def _parseValue(self, line, name):
-    match = re.search("\[%s = (.+?)\]" % name, line)
-    if match:
-      return match.group(1)
-    return None
-
-  def _parseLeakingTests(self):
-    leakingTests = []
-
-    for test in self.tests:
-      test["leakedWindows"] = [self.leakedWindows[id] for id in test["windows"] if id in self.leakedWindows]
-      test["leakedDocShells"] = [id for id in test["docShells"] if id in self.leakedDocShells]
-      test["leakCount"] = len(test["leakedWindows"]) + len(test["leakedDocShells"])
-
-      if test["leakCount"]:
-        leakingTests.append(test)
-
-    return sorted(leakingTests, key=itemgetter("leakCount"), reverse=True)
-
-  def _zipLeakedWindows(self, leakedWindows):
-    counts = []
-    counted = set()
-
-    for url in leakedWindows:
-      if not url in counted:
-        counts.append((url, leakedWindows.count(url)))
-        counted.add(url)
-
-    return sorted(counts, key=itemgetter(1), reverse=True)
--- a/build/mobile/remoteautomation.py
+++ b/build/mobile/remoteautomation.py
@@ -57,17 +57,17 @@ class RemoteAutomation(Automation):
         if crashreporter:
             env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
             env['MOZ_CRASHREPORTER'] = '1'
         else:
             env['MOZ_CRASHREPORTER_DISABLE'] = '1'
 
         return env
 
-    def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsDir, logger):
+    def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsDir):
         # maxTime is used to override the default timeout, we should honor that
         status = proc.wait(timeout = maxTime)
 
         print proc.stdout
 
         if (status == 1 and self._devicemanager.processExist(proc.procName)):
             # Then we timed out, make sure Fennec is dead
             proc.kill()
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -674,56 +674,44 @@ class Mochitest(object):
     # then again to actually run mochitest
     if options.timeout:
       timeout = options.timeout + 30
     elif not options.autorun:
       timeout = None
     else:
       timeout = 330.0 # default JS harness timeout is 300 seconds
 
-    # it's a debug build, we can parse leaked DOMWindows and docShells
-    # but skip for WebappRT chrome tests, where DOMWindow "leaks" aren't
-    # meaningful.  See https://bugzilla.mozilla.org/show_bug.cgi?id=733631#c46
-    if Automation.IS_DEBUG_BUILD and not options.webapprtChrome:
-      logger = ShutdownLeakLogger(self.automation.log)
-    else:
-      logger = None
-
     if options.vmwareRecording:
       self.startVMwareRecording(options);
 
     self.automation.log.info("INFO | runtests.py | Running tests: start.\n")
     try:
       status = self.automation.runApp(testURL, browserEnv, options.app,
                                   options.profilePath, options.browserArgs,
                                   runSSLTunnel = self.runSSLTunnel,
                                   utilityPath = options.utilityPath,
                                   xrePath = options.xrePath,
                                   certPath=options.certPath,
                                   debuggerInfo=debuggerInfo,
                                   symbolsPath=options.symbolsPath,
-                                  logger = logger,
                                   timeout = timeout)
     except KeyboardInterrupt:
       self.automation.log.info("INFO | runtests.py | Received keyboard interrupt.\n");
       status = -1
     except:
       self.automation.log.exception("INFO | runtests.py | Received unexpected exception while running application\n")
       status = 1
 
     if options.vmwareRecording:
       self.stopVMwareRecording();
 
     self.stopWebServer(options)
     self.stopWebSocketServer(options)
     processLeakLog(self.leak_report_file, options.leakThreshold)
 
-    if logger:
-      logger.parse()
-
     self.automation.log.info("\nINFO | runtests.py | Running tests: end.")
 
     if manifest is not None:
       self.cleanup(manifest, options)
     return status
 
   def makeTestConfig(self, options):
     "Creates a test configuration file for customizing test execution."