Backed out changeset 1c19d2a03d90 (bug 827446) for packaging bustage on a CLOSED TREE.
authorRyan VanderMeulen <ryanvm@gmail.com>
Wed, 13 Mar 2013 09:22:00 -0400
changeset 135060 0a02276859d1d0d90755641330db0a6b91d5966e
parent 135059 d50cbeebba114de414996b7de2851ef3edabcb88
child 135061 3c3f560a2a5caa2c4171c3baa2ab3e5c188c15dd
push id2452
push userlsblakk@mozilla.com
push dateMon, 13 May 2013 16:59:38 +0000
treeherdermozilla-beta@d4b152d29d8d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs827446
milestone22.0a1
backs out1c19d2a03d900cef52914ff4c9d4306bd173c508
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 1c19d2a03d90 (bug 827446) for packaging bustage on a CLOSED TREE.
build/Makefile.in
build/automation.py.in
build/automationutils.py
build/mobile/remoteautomation.py
testing/mozbase/Makefile.in
testing/xpcshell/runxpcshelltests.py
--- a/build/Makefile.in
+++ b/build/Makefile.in
@@ -86,16 +86,17 @@ PYTHON_UNIT_TESTS := \
 
 include $(topsrcdir)/config/rules.mk
 
 # we install to _leaktest/
 TARGET_DEPTH = ..
 include $(srcdir)/automation-build.mk
 
 _LEAKTEST_DIR = $(DEPTH)/_leaktest
+GARBAGE_DIRS += $(_LEAKTEST_DIR)
 
 _LEAKTEST_FILES =    \
 		automation.py \
 		automationutils.py \
 		leaktest.py \
 		bloatcycle.html \
 		$(topsrcdir)/build/pgo/server-locations.txt \
 		$(topsrcdir)/build/pgo/favicon.ico \
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -19,34 +19,16 @@ import threading
 import tempfile
 import sqlite3
 from string import Template
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 sys.path.insert(0, SCRIPT_DIR)
 import automationutils
 
-# --------------------------------------------------------------
-# TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
-#
-here = os.path.dirname(__file__)
-mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))
-
-try:
-    import mozcrash
-except:
-    deps = ['mozcrash',
-            'mozlog']
-    for dep in deps:
-        module = os.path.join(mozbase, dep)
-        if module not in sys.path:
-            sys.path.append(module)
-    import mozcrash
-# ---------------------------------------------------------------
-
 _DEFAULT_WEB_SERVER = "127.0.0.1"
 _DEFAULT_HTTP_PORT = 8888
 _DEFAULT_SSL_PORT = 4443
 _DEFAULT_WEBSOCKET_PORT = 9988
 
 # from nsIPrincipal.idl
 _APP_STATUS_NOT_INSTALLED = 0
 _APP_STATUS_INSTALLED     = 1
@@ -1144,17 +1126,17 @@ user_pref("camino.use_system_proxy_setti
       self.log.info("INFO | automation.py | Checking for orphan process with PID: %d", processPID)
       if self.isPidAlive(processPID):
         foundZombie = True
         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | child process %d still alive after shutdown", processPID)
         self.killPid(processPID)
     return foundZombie
 
   def checkForCrashes(self, profileDir, symbolsPath):
-    return mozcrash.check_for_crashes(os.path.join(profileDir, "minidumps"), symbolsPath, test_name=self.lastTestSeen)
+    return automationutils.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath, self.lastTestSeen)
 
   def runApp(self, testURL, env, app, profileDir, extraArgs,
              runSSLTunnel = False, utilityPath = None,
              xrePath = None, certPath = None,
              debuggerInfo = None, symbolsPath = None,
              timeout = -1, maxTime = None, onLaunch = None):
     """
     Run the app, log the duration it took to execute, return the status code.
--- a/build/automationutils.py
+++ b/build/automationutils.py
@@ -6,16 +6,17 @@
 from __future__ import with_statement
 import glob, logging, os, platform, shutil, subprocess, sys, tempfile, urllib2, zipfile
 import re
 from urlparse import urlparse
 
 __all__ = [
   "ZipFileReader",
   "addCommonOptions",
+  "checkForCrashes",
   "dumpLeakLog",
   "isURL",
   "processLeakLog",
   "getDebuggerInfo",
   "DEBUGGER_INFO",
   "replaceBackSlashes",
   "wrapCommand",
   ]
@@ -126,16 +127,105 @@ def addCommonOptions(parser, defaults={}
                     action = "store", dest = "debuggerArgs",
                     help = "pass the given args to the debugger _before_ "
                            "the application on the command line")
   parser.add_option("--debugger-interactive",
                     action = "store_true", dest = "debuggerInteractive",
                     help = "prevents the test harness from redirecting "
                         "stdout and stderr for interactive debuggers")
 
+def checkForCrashes(dumpDir, symbolsPath, testName=None):
+  stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
+  # try to get the caller's filename if no test name is given
+  if testName is None:
+    try:
+      testName = os.path.basename(sys._getframe(1).f_code.co_filename)
+    except:
+      testName = "unknown"
+
+  # Check preconditions
+  dumps = glob.glob(os.path.join(dumpDir, '*.dmp'))
+  if len(dumps) == 0:
+    return False
+
+  try:
+    removeSymbolsPath = False
+
+    # If our symbols are at a remote URL, download them now
+    if symbolsPath and isURL(symbolsPath):
+      print "Downloading symbols from: " + symbolsPath
+      removeSymbolsPath = True
+      # Get the symbols and write them to a temporary zipfile
+      data = urllib2.urlopen(symbolsPath)
+      symbolsFile = tempfile.TemporaryFile()
+      symbolsFile.write(data.read())
+      # extract symbols to a temporary directory (which we'll delete after
+      # processing all crashes)
+      symbolsPath = tempfile.mkdtemp()
+      zfile = ZipFileReader(symbolsFile)
+      zfile.extractall(symbolsPath)
+
+    for d in dumps:
+      stackwalkOutput = []
+      stackwalkOutput.append("Crash dump filename: " + d)
+      topFrame = None
+      if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
+        # run minidump stackwalk
+        p = subprocess.Popen([stackwalkPath, d, symbolsPath],
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        (out, err) = p.communicate()
+        if len(out) > 3:
+          # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
+          stackwalkOutput.append(out)
+          # The top frame of the crash is always the line after "Thread N (crashed)"
+          # Examples:
+          #  0  libc.so + 0xa888
+          #  0  libnss3.so!nssCertificate_Destroy [certificate.c : 102 + 0x0]
+          #  0  mozjs.dll!js::GlobalObject::getDebuggers() [GlobalObject.cpp:89df18f9b6da : 580 + 0x0]
+          #  0  libxul.so!void js::gc::MarkInternal<JSObject>(JSTracer*, JSObject**) [Marking.cpp : 92 + 0x28]
+          lines = out.splitlines()
+          for i, line in enumerate(lines):
+            if "(crashed)" in line:
+              match = re.search(r"^ 0  (?:.*!)?(?:void )?([^\[]+)", lines[i+1])
+              if match:
+                topFrame = "@ %s" % match.group(1).strip()
+              break
+        else:
+          stackwalkOutput.append("stderr from minidump_stackwalk:")
+          stackwalkOutput.append(err)
+        if p.returncode != 0:
+          stackwalkOutput.append("minidump_stackwalk exited with return code %d" % p.returncode)
+      else:
+        if not symbolsPath:
+          stackwalkOutput.append("No symbols path given, can't process dump.")
+        if not stackwalkPath:
+          stackwalkOutput.append("MINIDUMP_STACKWALK not set, can't process dump.")
+        elif stackwalkPath and not os.path.exists(stackwalkPath):
+          stackwalkOutput.append("MINIDUMP_STACKWALK binary not found: %s" % stackwalkPath)
+      if not topFrame:
+        topFrame = "Unknown top frame"
+      log.info("PROCESS-CRASH | %s | application crashed [%s]", testName, topFrame)
+      print '\n'.join(stackwalkOutput)
+      dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
+      if dumpSavePath:
+        shutil.move(d, dumpSavePath)
+        print "Saved dump as %s" % os.path.join(dumpSavePath,
+                                                os.path.basename(d))
+      else:
+        os.remove(d)
+      extra = os.path.splitext(d)[0] + ".extra"
+      if os.path.exists(extra):
+        os.remove(extra)
+  finally:
+    if removeSymbolsPath:
+      shutil.rmtree(symbolsPath)
+
+  return True
+
 def getFullPath(directory, path):
   "Get an absolute path relative to 'directory'."
   return os.path.normpath(os.path.join(directory, os.path.expanduser(path)))
 
 def searchPath(directory, path):
   "Go one step beyond getFullPath and try the various folders in PATH"
   # Try looking in the current working directory first.
   newpath = getFullPath(directory, path)
--- a/build/mobile/remoteautomation.py
+++ b/build/mobile/remoteautomation.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import time
 import re
 import os
+import automationutils
 import tempfile
 import shutil
 import subprocess
 
 from automation import Automation
 from devicemanager import NetworkTools, DMError
 
 # signatures for logcat messages that we don't care about much
@@ -127,18 +128,18 @@ class RemoteAutomation(Automation):
             if not self._devicemanager.dirExists(remoteCrashDir):
                 # As of this writing, the minidumps directory is automatically
                 # created when fennec (first) starts, so its lack of presence
                 # is a hint that something went wrong.
                 print "Automation Error: No crash directory (%s) found on remote device" % remoteCrashDir
                 # Whilst no crash was found, the run should still display as a failure
                 return True
             self._devicemanager.getDirectory(remoteCrashDir, dumpDir)
-            crashed = Automation.checkForCrashes(self, dumpDir, symbolsPath)
-
+            crashed = automationutils.checkForCrashes(dumpDir, symbolsPath,
+                                            self.lastTestSeen)
         finally:
             try:
                 shutil.rmtree(dumpDir)
             except:
                 print "WARNING: unable to remove directory: %s" % dumpDir
         return crashed
 
     def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
--- a/testing/mozbase/Makefile.in
+++ b/testing/mozbase/Makefile.in
@@ -13,17 +13,16 @@ MODULE = testing_mozbase
 
 include $(topsrcdir)/config/rules.mk
 
 # Harness packages from the srcdir;
 # python packages to be installed IN INSTALLATION ORDER.
 # Packages later in the list can depend only on packages earlier in the list.
 MOZBASE_PACKAGES = \
   manifestdestiny \
-  mozcrash \
   mozfile \
   mozhttpd \
   mozinfo \
   mozinstall \
   mozlog \
   mozprocess \
   mozprofile \
   mozrunner \
--- a/testing/xpcshell/runxpcshelltests.py
+++ b/testing/xpcshell/runxpcshelltests.py
@@ -13,34 +13,16 @@ from tempfile import mkdtemp, gettempdir
 import manifestparser
 import mozinfo
 import random
 import socket
 import time
 
 from automationutils import *
 
-# --------------------------------------------------------------
-# TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
-#
-here = os.path.dirname(__file__)
-mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))
-
-try:
-    import mozcrash
-except:
-    deps = ['mozcrash',
-            'mozlog']
-    for dep in deps:
-        module = os.path.join(mozbase, dep)
-        if module not in sys.path:
-            sys.path.append(module)
-    import mozcrash
-# ---------------------------------------------------------------
-
 #TODO: replace this with json.loads when Python 2.6 is required.
 def parse_json(j):
     """
     Awful hack to parse a restricted subset of JSON strings into Python dicts.
     """
     return eval(j, {'true':True,'false':False,'null':None})
 
 """ Control-C handling """
@@ -907,17 +889,17 @@ class XPCShellTests(object):
                     xunitResult["passed"] = True
 
                     if expected:
                         self.passCount += 1
                     else:
                         self.todoCount += 1
                         xunitResult["todo"] = True
 
-                if mozcrash.check_for_crashes(testdir, self.symbolsPath, test_name=name):
+                if checkForCrashes(testdir, self.symbolsPath, testName=name):
                     message = "PROCESS-CRASH | %s | application crashed" % name
                     self.failCount += 1
                     xunitResult["passed"] = False
                     xunitResult["failure"] = {
                       "type": "PROCESS-CRASH",
                       "message": message,
                       "text": stdout
                     }