Bug 481732 - Check for crash minidumps in unit tests and dump them, if the MINIDUMP_STACKWALK path is set in the environment, r=ted
authorBenjamin Smedberg <benjamin@smedbergs.us>
Tue, 10 Mar 2009 13:36:14 -0400
changeset 25988 a2a7177a218e5541bf52d406d11517ce6e9c5022
parent 25987 d85e2a66d9408ba025af19b8d3162618f6f789e8
child 25989 c3d89722f165e6a14f29f95ba4c81e63d27009a3
push idunknown
push userunknown
push dateunknown
reviewersted
bugs481732
milestone1.9.2a1pre
Bug 481732 - Check for crash minidumps in unit tests and dump them, if the MINIDUMP_STACKWALK path is set in the environment, r=ted
Makefile.in
build/automation-build.mk
build/automation.py.in
layout/tools/reftest/runreftest.py
testing/mochitest/runtests.py.in
--- a/Makefile.in
+++ b/Makefile.in
@@ -185,28 +185,29 @@ EXTRA_BUILDID := -$(MOZ_SYMBOLS_EXTRA_BU
 endif
 
 SYMBOL_INDEX_NAME = \
   $(MOZ_APP_NAME)-$(MOZ_APP_VERSION)-$(OS_ARCH)-$(BUILDID)$(EXTRA_BUILDID)-symbols.txt
 
 buildsymbols:
 ifdef MOZ_CRASHREPORTER
 	echo building symbol store
-	mkdir -p $(DIST)/crashreporter-symbols/$(BUILDID)
+	$(RM) -rf $(DIST)/crashreporter-symbols
+	$(NSINSTALL) -D $(DIST)/crashreporter-symbols
 	$(PYTHON) $(topsrcdir)/toolkit/crashreporter/tools/symbolstore.py \
 	  $(MAKE_SYM_STORE_ARGS)                                          \
 	  $(foreach dir,$(SYM_STORE_SOURCE_DIRS),-s $(dir))               \
 	  $(DUMP_SYMS_BIN)                                                \
-	  $(DIST)/crashreporter-symbols/$(BUILDID)                        \
+	  $(DIST)/crashreporter-symbols                                   \
 	  $(MAKE_SYM_STORE_PATH) >                                        \
-	  $(DIST)/crashreporter-symbols/$(BUILDID)/$(SYMBOL_INDEX_NAME)
+	  $(DIST)/crashreporter-symbols/$(SYMBOL_INDEX_NAME)
 	echo packing symbols
 	mkdir -p $(topsrcdir)/../$(BUILDID)
-	cd $(DIST)/crashreporter-symbols/$(BUILDID) && \
-          zip -r9D ../../$(SYMBOL_ARCHIVE_BASENAME).zip .
+	cd $(DIST)/crashreporter-symbols && \
+          zip -r9D ../$(SYMBOL_ARCHIVE_BASENAME).zip .
 endif # MOZ_CRASHREPORTER
 
 uploadsymbols:
 ifdef MOZ_CRASHREPORTER
 	$(topsrcdir)/toolkit/crashreporter/tools/upload_symbols.sh $(DIST)/$(SYMBOL_ARCHIVE_BASENAME).zip
 endif
 
 ifeq ($(OS_ARCH),WINNT)
--- a/build/automation-build.mk
+++ b/build/automation-build.mk
@@ -17,32 +17,34 @@ else
 browser_path = \"$(TARGET_DIST)/$(MOZ_APP_DISPLAYNAME).app/Contents/MacOS/$(PROGRAM)\"
 endif
 else
 browser_path = \"$(TARGET_DIST)/bin/$(PROGRAM)\"
 endif
 endif
 
 _PROFILE_DIR = $(TARGET_DEPTH)/_profile/pgo
+_SYMBOLS_PATH = $(TARGET_DIST)/crashreporter-symbols
 
 ifneq (,$(filter /%,$(topsrcdir)))
 # $(topsrcdir) is already an absolute pathname.
 ABSOLUTE_TOPSRCDIR = $(topsrcdir)
 else
 # $(topsrcdir) is a relative pathname: prepend the current directory.
 ABSOLUTE_TOPSRCDIR = $(CURDIR)/$(topsrcdir)
 endif
 _CERTS_SRC_DIR = $(ABSOLUTE_TOPSRCDIR)/build/pgo/certs
 
 AUTOMATION_PPARGS = 	\
 			-DBROWSER_PATH=$(browser_path) \
 			-DXPC_BIN_PATH=\"$(LIBXUL_DIST)/bin\" \
 			-DBIN_SUFFIX=\"$(BIN_SUFFIX)\" \
 			-DPROFILE_DIR=\"$(_PROFILE_DIR)\" \
 			-DCERTS_SRC_DIR=\"$(_CERTS_SRC_DIR)\" \
+			-DSYMBOLS_PATH=\"$(_SYMBOLS_PATH)\" \
 			$(NULL)
 
 ifeq ($(OS_ARCH),Darwin)
 AUTOMATION_PPARGS += -DIS_MAC=1
 else
 AUTOMATION_PPARGS += -DIS_MAC=0
 endif
 
@@ -63,11 +65,11 @@ AUTOMATION_PPARGS += -DIS_TEST_BUILD=0
 endif
 
 ifeq ($(MOZ_DEBUG), 1)
 AUTOMATION_PPARGS += -DIS_DEBUG_BUILD=1
 else
 AUTOMATION_PPARGS += -DIS_DEBUG_BUILD=0
 endif
 
-automation.py: $(topsrcdir)/build/automation.py.in
+automation.py: $(topsrcdir)/build/automation.py.in $(topsrcdir)/build/automation-build.mk
 	$(PYTHON) $(topsrcdir)/config/Preprocessor.py \
 	$(AUTOMATION_PPARGS) $(DEFINES) $(ACDEFINES) $< > $@
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -43,16 +43,17 @@ import itertools
 import logging
 import os
 import re
 import shutil
 import signal
 import subprocess
 import sys
 import threading
+import glob
 
 """
 Runs the browser from a script, and provides useful utilities
 for setting up the browser environment.
 """
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 
@@ -82,16 +83,17 @@ IS_CYGWIN = False
 #expand BIN_SUFFIX = __BIN_SUFFIX__
 
 UNIXISH = not IS_WIN32 and not IS_MAC
 
 #expand DEFAULT_APP = "./" + __BROWSER_PATH__
 #expand CERTS_SRC_DIR = __CERTS_SRC_DIR__
 #expand IS_TEST_BUILD = __IS_TEST_BUILD__
 #expand IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
+#expand SYMBOLS_PATH = __SYMBOLS_PATH__
 
 ###########
 # LOGGING #
 ###########
 
 # We use the logging system here primarily because it'll handle multiple
 # threads, which is needed to process the output of the server and application
 # processes simultaneously.
@@ -396,18 +398,37 @@ def environment(env = None, xrePath = DI
     if IS_MAC:
       envVar = "DYLD_LIBRARY_PATH"
     if envVar in env:
       ldLibraryPath = ldLibraryPath + ":" + env[envVar]
     env[envVar] = ldLibraryPath
   elif IS_WIN32:
     env["PATH"] = env["PATH"] + ";" + ldLibraryPath
 
+  env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
+  env['MOZ_CRASHREPORTER'] = '1'
+
   return env
 
+def checkForCrashes(profileDir, symbolsPath):
+  stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
+
+  foundCrash = False
+  dumps = glob.glob(os.path.join(profileDir, 'minidumps', '*.dmp'))
+  for d in dumps:
+    log.info("TEST-UNEXPECTED-FAIL | (automation.py) | Browser crashed (minidump found)")
+    if symbolsPath and stackwalkPath:
+      nullfd = open(os.devnull, 'w')
+      # eat minidump_stackwalk errors
+      subprocess.call([stackwalkPath, d, symbolsPath], stderr=nullfd)
+      nullfd.close()
+    os.remove(d)
+    foundCrash = True
+  return foundCrash
+
 ###############
 # RUN THE APP #
 ###############
 
 def processLeakLog(leakLogFile, leakThreshold):
   "Process the leak log."
 
   if not os.path.exists(leakLogFile):
@@ -475,17 +496,20 @@ def processLeakLog(leakLogFile, leakThre
                      "instance": instance,
                      "name": name,
                      "size": matches.group("size"),
                      "rest": rest })
     if not seenTotal:
       log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | missing output line for total leaks!")
     leaks.close()
 
-def runApp(testURL, env, app, profileDir, extraArgs, runSSLTunnel = False, utilityPath = DIST_BIN, xrePath = DIST_BIN, certPath = CERTS_SRC_DIR, debuggerInfo = None):
+def runApp(testURL, env, app, profileDir, extraArgs,
+           runSSLTunnel = False, utilityPath = DIST_BIN,
+           xrePath = DIST_BIN, certPath = CERTS_SRC_DIR,
+           debuggerInfo = None, symbolsPath = SYMBOLS_PATH):
   "Run the app, log the duration it took to execute, return the status code."
 
   if IS_TEST_BUILD and runSSLTunnel:
     # create certificate database for the profile
     certificateStatus = fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
     if certificateStatus != 0:
       log.info("TEST-UNEXPECTED FAIL | (automation.py) | Certificate integration failed")
       return certificateStatus
@@ -543,12 +567,15 @@ def runApp(testURL, env, app, profileDir
       log.info(line.rstrip())
       line = proc.stdout.readline()
 
   status = proc.wait()
   if status != 0:
     log.info("TEST-UNEXPECTED-FAIL | (automation.py) | Exited with code %d during test run", status)
   log.info("INFO | (automation.py) | Application ran for: %s", str(datetime.now() - startTime))
 
+  if checkForCrashes(profileDir, symbolsPath):
+    status = -1
+
   if IS_TEST_BUILD and runSSLTunnel:
     ssltunnelProcess.kill()
 
   return status
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -79,16 +79,20 @@ def main():
   parser.add_option("--appname",
                     action = "store", type = "string", dest = "app",
                     default = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP),
                     help = "absolute path to application, overriding default")
   parser.add_option("--extra-profile-file",
                     action = "append", dest = "extraProfileFiles",
                     default = [],
                     help = "copy specified files/dirs to testing profile")
+  parser.add_option("--symbols-path",
+                    action = "store", type = "string", dest = "symbolsPath",
+                    default = automation.SYMBOLS_PATH,
+                    help = "absolute path to directory containing breakpad symbols")
   options, args = parser.parse_args()
 
   if len(args) != 1:
     print >>sys.stderr, "No reftest.list specified."
     sys.exit(1)
 
   options.app = getFullPath(options.app)
   if not os.path.exists(options.app):
@@ -131,17 +135,18 @@ Are you executing $objdir/_tests/reftest
         # For the time being, simply copy the log. (Bug 469518)
         log.info(leaks.read().rstrip())
         leaks.close()
 
     # run once with -silent to let the extension manager do its thing
     # and then exit the app
     log.info("REFTEST INFO | runreftest.py | Performing extension manager registration: start.\n")
     status = automation.runApp(None, browserEnv, options.app, profileDir,
-                               extraArgs = ["-silent"])
+                               extraArgs = ["-silent"],
+                               symbolsPath=options.symbolsPath)
     # We don't care to call |processLeakLog()| for this step.
     log.info("\nREFTEST INFO | runreftest.py | Performing extension manager registration: end.")
 
     # then again to actually run reftest
     log.info("REFTEST INFO | runreftest.py | Running tests: start.\n")
     reftestlist = getFullPath(args[0])
     status = automation.runApp(None, browserEnv, options.app, profileDir,
                                extraArgs = ["-reftest", reftestlist])
--- a/testing/mochitest/runtests.py.in
+++ b/testing/mochitest/runtests.py.in
@@ -131,16 +131,21 @@ class MochitestOptions(optparse.OptionPa
     # we'll default this to the directory of app below
     defaults["xrePath"] = None
 
     self.add_option("--utility-path",
                     action = "store", type = "string", dest = "utilityPath",
                     help = "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)")
     defaults["utilityPath"] = automation.DIST_BIN
 
+    self.add_option("--symbols-path",
+                    action = "store", type = "string", dest = "symbolsPath",
+                    help = "absolute path to directory containing breakpad symbols")
+    defaults["symbolsPath"] = automation.SYMBOLS_PATH
+
     self.add_option("--certificate-path",
                     action = "store", type = "string", dest = "certPath",
                     help = "absolute path to directory containing certificate store to use testing profile")
     defaults["certPath"] = automation.CERTS_SRC_DIR
 
     self.add_option("--log-file",
                     action = "store", type = "string",
                     dest = "logFile", metavar = "FILE",
@@ -467,17 +472,18 @@ Are you executing $objdir/_tests/testing
     browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 
   status = automation.runApp(testURL, browserEnv, options.app,
                              PROFILE_DIRECTORY, options.browserArgs,
                              runSSLTunnel = True,
                              utilityPath = options.utilityPath,
                              xrePath = options.xrePath,
                              certPath=options.certPath,
-                             debuggerInfo=debuggerInfo)
+                             debuggerInfo=debuggerInfo,
+                             symbolsPath=options.symbolsPath)
 
   # Server's no longer needed, and perhaps more importantly, anything it might
   # spew to console shouldn't disrupt the leak information table we print next.
   server.stop()
 
   automation.processLeakLog(LEAK_REPORT_FILE, options.leakThreshold)
 
   # delete the profile and manifest