Bug 679759 - MINIDUMP_STACKWALK_CGI support, let harness download symbols as needed, r=ted
☠☠ backed out by 72592670532d ☠ ☠
authorWilliam Lachance <wlachance@mozilla.com>
Mon, 07 Nov 2011 13:14:22 -0800
changeset 80593 cd695cdb3b4fb6b09a66ee9b23fd74a425f3ff6b
parent 80592 08819514d15922672f1019889202447350362c76
child 80594 375aaf23cc2fe652e79d5b3a5a363d2963e79006
push id506
push userclegnitto@mozilla.com
push dateWed, 09 Nov 2011 02:03:18 +0000
treeherdermozilla-aurora@63587fc7bb93 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs679759
milestone10.0a1
Bug 679759 - MINIDUMP_STACKWALK_CGI support, let harness download symbols as needed, r=ted
build/automation.py.in
build/automationutils.py
layout/tools/reftest/Makefile.in
testing/mochitest/Makefile.in
testing/xpcshell/Makefile.in
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -47,17 +47,16 @@ import re
 import select
 import shutil
 import signal
 import subprocess
 import sys
 import threading
 import tempfile
 import sqlite3
-import zipfile
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 sys.path.insert(0, SCRIPT_DIR)
 import automationutils
 
 _DEFAULT_WEB_SERVER = "127.0.0.1"
 _DEFAULT_HTTP_PORT = 8888
 _DEFAULT_SSL_PORT = 4443
@@ -93,79 +92,16 @@ else:
 # threads, which is needed to process the output of the server and application
 # processes simultaneously.
 _log = logging.getLogger()
 handler = logging.StreamHandler(sys.stdout)
 _log.setLevel(logging.INFO)
 _log.addHandler(handler)
 
 
-class ZipFileReader(object):
-  """
-  Class to read zip files in Python 2.5 and later. Limited to only what we
-  actually use.
-  """
-
-  def __init__(self, filename):
-    self._zipfile = zipfile.ZipFile(filename, "r")
-
-  def __del__(self):
-    self._zipfile.close()
-
-  def _getnormalizedpath(self, path):
-    """
-    Gets a normalized path from 'path' (or the current working directory if
-    'path' is None). Also asserts that the path exists.
-    """
-    if path is None:
-      path = os.curdir
-    path = os.path.normpath(os.path.expanduser(path))
-    assert os.path.isdir(path)
-    return path
-
-  def _extractname(self, name, path):
-    """
-    Extracts a file with the given name from the zip file to the given path.
-    Also creates any directories needed along the way.
-    """
-    filename = os.path.normpath(os.path.join(path, name))
-    if name.endswith("/"):
-      os.makedirs(filename)
-    else:
-      path = os.path.split(filename)[0]
-      if not os.path.isdir(path):
-        os.makedirs(path)
-      with open(filename, "wb") as dest:
-        dest.write(self._zipfile.read(name))
-
-  def namelist(self):
-    return self._zipfile.namelist()
-
-  def read(self, name):
-    return self._zipfile.read(name)
-
-  def extract(self, name, path = None):
-    if hasattr(self._zipfile, "extract"):
-      return self._zipfile.extract(name, path)
-
-    # This will throw if name is not part of the zip file.
-    self._zipfile.getinfo(name)
-
-    self._extractname(name, self._getnormalizedpath(path))
-
-  def extractall(self, path = None):
-    if hasattr(self._zipfile, "extractall"):
-      return self._zipfile.extractall(path)
-
-    path = self._getnormalizedpath(path)
-
-    for name in self._zipfile.namelist():
-      self._extractname(name, path)
-
-
 #################
 # PROFILE SETUP #
 #################
 
 class SyntaxError(Exception):
   "Signifies a syntax error on a particular line in server-locations.txt."
 
   def __init__(self, lineno, msg = None):
@@ -1047,17 +983,17 @@ user_pref("camino.use_system_proxy_setti
 
     installRDFFilename = "install.rdf"
 
     extensionsRootDir = os.path.join(profileDir, "extensions", "staged")
     if not os.path.isdir(extensionsRootDir):
       os.makedirs(extensionsRootDir)
 
     if os.path.isfile(extensionSource):
-      reader = ZipFileReader(extensionSource)
+      reader = automationutils.ZipFileReader(extensionSource)
 
       for filename in reader.namelist():
         # Sanity check the zip file.
         if os.path.isabs(filename):
           self.log.info("INFO | automation.py | Cannot install extension, bad files in xpi")
           return
 
         # We may need to dig the extensionID out of the zip file...
--- a/build/automationutils.py
+++ b/build/automationutils.py
@@ -31,21 +31,22 @@
 # use your version of this file under the terms of the MPL, indicate your
 # decision by deleting the provisions above and replace them with the notice
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK ***** */
 
-import glob, logging, os, platform, shutil, subprocess, sys
+import glob, logging, os, platform, shutil, subprocess, sys, tempfile, urllib2, zipfile
 import re
 from urlparse import urlparse
 
 __all__ = [
+  "ZipFileReader",
   "addCommonOptions",
   "checkForCrashes",
   "dumpLeakLog",
   "isURL",
   "processLeakLog",
   "getDebuggerInfo",
   "DEBUGGER_INFO",
   "replaceBackSlashes",
@@ -65,16 +66,78 @@ DEBUGGER_INFO = {
   # valgrind doesn't explain much about leaks unless you set the
   # '--leak-check=full' flag.
   "valgrind": {
     "interactive": False,
     "args": "--leak-check=full"
   }
 }
 
+class ZipFileReader(object):
+  """
+  Class to read zip files in Python 2.5 and later. Limited to only what we
+  actually use.
+  """
+
+  def __init__(self, filename):
+    self._zipfile = zipfile.ZipFile(filename, "r")
+
+  def __del__(self):
+    self._zipfile.close()
+
+  def _getnormalizedpath(self, path):
+    """
+    Gets a normalized path from 'path' (or the current working directory if
+    'path' is None). Also asserts that the path exists.
+    """
+    if path is None:
+      path = os.curdir
+    path = os.path.normpath(os.path.expanduser(path))
+    assert os.path.isdir(path)
+    return path
+
+  def _extractname(self, name, path):
+    """
+    Extracts a file with the given name from the zip file to the given path.
+    Also creates any directories needed along the way.
+    """
+    filename = os.path.normpath(os.path.join(path, name))
+    if name.endswith("/"):
+      os.makedirs(filename)
+    else:
+      path = os.path.split(filename)[0]
+      if not os.path.isdir(path):
+        os.makedirs(path)
+      with open(filename, "wb") as dest:
+        dest.write(self._zipfile.read(name))
+
+  def namelist(self):
+    return self._zipfile.namelist()
+
+  def read(self, name):
+    return self._zipfile.read(name)
+
+  def extract(self, name, path = None):
+    if hasattr(self._zipfile, "extract"):
+      return self._zipfile.extract(name, path)
+
+    # This will throw if name is not part of the zip file.
+    self._zipfile.getinfo(name)
+
+    self._extractname(name, self._getnormalizedpath(path))
+
+  def extractall(self, path = None):
+    if hasattr(self._zipfile, "extractall"):
+      return self._zipfile.extractall(path)
+
+    path = self._getnormalizedpath(path)
+
+    for name in self._zipfile.namelist():
+      self._extractname(name, path)
+
 log = logging.getLogger()
 
 def isURL(thing):
   """Return True if |thing| looks like a URL."""
   return urlparse(thing).scheme != ''
 
 def addCommonOptions(parser, defaults={}):
   parser.add_option("--xre-path",
@@ -97,88 +160,87 @@ def addCommonOptions(parser, defaults={}
                            "the application on the command line")
   parser.add_option("--debugger-interactive",
                     action = "store_true", dest = "debuggerInteractive",
                     help = "prevents the test harness from redirecting "
                         "stdout and stderr for interactive debuggers")
 
 def checkForCrashes(dumpDir, symbolsPath, testName=None):
   stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
-  stackwalkCGI = os.environ.get('MINIDUMP_STACKWALK_CGI', None)
   # try to get the caller's filename if no test name is given
   if testName is None:
     try:
       testName = os.path.basename(sys._getframe(1).f_code.co_filename)
     except:
       testName = "unknown"
 
-  foundCrash = False
+  # Check preconditions
   dumps = glob.glob(os.path.join(dumpDir, '*.dmp'))
-  for d in dumps:
-    log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
-    print "Crash dump filename: " + d
-    if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
-      p = subprocess.Popen([stackwalkPath, d, symbolsPath],
-                           stdout=subprocess.PIPE,
-                           stderr=subprocess.PIPE)
-      (out, err) = p.communicate()
-      if len(out) > 3:
-        # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
-        print out
+  if len(dumps) == 0:
+    return False
+
+  foundCrash = False
+  removeSymbolsPath = False
+
+  # If our symbols are at a remote URL, download them now
+  if isURL(symbolsPath):
+    print "Downloading symbols from: " + symbolsPath
+    removeSymbolsPath = True
+    # Get the symbols and write them to a temporary zipfile
+    data = urllib2.urlopen(symbolsPath)
+    symbolsFile = tempfile.TemporaryFile()
+    symbolsFile.write(data.read())
+    # extract symbols to a temporary directory (which we'll delete after
+    # processing all crashes)
+    symbolsPath = tempfile.mkdtemp()
+    zfile = ZipFileReader(symbolsFile)
+    zfile.extractall(symbolsPath)
+
+  try:
+    for d in dumps:
+      log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
+      print "Crash dump filename: " + d
+      if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
+        # run minidump stackwalk
+        p = subprocess.Popen([stackwalkPath, d, symbolsPath],
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        (out, err) = p.communicate()
+        if len(out) > 3:
+          # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
+          print out
+        else:
+          print "stderr from minidump_stackwalk:"
+          print err
+        if p.returncode != 0:
+          print "minidump_stackwalk exited with return code %d" % p.returncode
       else:
-        print "stderr from minidump_stackwalk:"
-        print err
-      if p.returncode != 0:
-        print "minidump_stackwalk exited with return code %d" % p.returncode
-    elif stackwalkCGI and symbolsPath and isURL(symbolsPath):
-      f = None
-      try:
-        f = open(d, "rb")
-        sys.path.append(os.path.join(os.path.dirname(__file__), "poster.zip"))
-        from poster.encode import multipart_encode
-        from poster.streaminghttp import register_openers
-        import urllib2
-        register_openers()
-        datagen, headers = multipart_encode({"minidump": f,
-                                             "symbols": symbolsPath})
-        request = urllib2.Request(stackwalkCGI, datagen, headers)
-        result = urllib2.urlopen(request).read()
-        if len(result) > 3:
-          print result
-        else:
-          print "stackwalkCGI returned nothing."
-      finally:
-        if f:
-          f.close()
-    else:
-      if not symbolsPath:
-        print "No symbols path given, can't process dump."
-      if not stackwalkPath and not stackwalkCGI:
-        print "Neither MINIDUMP_STACKWALK nor MINIDUMP_STACKWALK_CGI is set, can't process dump."
+        if not symbolsPath:
+          print "No symbols path given, can't process dump."
+        if not stackwalkPath:
+          print "MINIDUMP_STACKWALK not set, can't process dump."
+        elif stackwalkPath and not os.path.exists(stackwalkPath):
+          print "MINIDUMP_STACKWALK binary not found: %s" % stackwalkPath
+      dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
+      if dumpSavePath:
+        shutil.move(d, dumpSavePath)
+        print "Saved dump as %s" % os.path.join(dumpSavePath,
+                                                os.path.basename(d))
       else:
-        if stackwalkPath and not os.path.exists(stackwalkPath):
-          print "MINIDUMP_STACKWALK binary not found: %s" % stackwalkPath
-        elif stackwalkCGI and not isURL(stackwalkCGI):
-          print "MINIDUMP_STACKWALK_CGI is not a URL: %s" % stackwalkCGI
-        elif symbolsPath and not isURL(symbolsPath):
-          print "symbolsPath is not a URL: %s" % symbolsPath
-    dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
-    if dumpSavePath:
-      shutil.move(d, dumpSavePath)
-      print "Saved dump as %s" % os.path.join(dumpSavePath,
-                                              os.path.basename(d))
-    else:
-      os.remove(d)
-    extra = os.path.splitext(d)[0] + ".extra"
-    if os.path.exists(extra):
-      os.remove(extra)
-    foundCrash = True
+        os.remove(d)
+      extra = os.path.splitext(d)[0] + ".extra"
+      if os.path.exists(extra):
+        os.remove(extra)
+      foundCrash = True
+  finally:
+    if removeSymbolsPath:
+      shutil.rmtree(symbolsPath)
 
   return foundCrash
-  
+
 def getFullPath(directory, path):
   "Get an absolute path relative to 'directory'."
   return os.path.normpath(os.path.join(directory, os.path.expanduser(path)))
 
 def searchPath(directory, path):
   "Go one step beyond getFullPath and try the various folders in PATH"
   # Try looking in the current working directory first.
   newpath = getFullPath(directory, path)
--- a/layout/tools/reftest/Makefile.in
+++ b/layout/tools/reftest/Makefile.in
@@ -79,17 +79,16 @@ endif
 _HARNESS_FILES = \
   $(srcdir)/runreftest.py \
   $(srcdir)/remotereftest.py \
   automation.py \
   $(topsrcdir)/build/mobile/devicemanager.py \
   $(topsrcdir)/build/mobile/devicemanagerADB.py \
   $(topsrcdir)/build/mobile/devicemanagerSUT.py \
   $(topsrcdir)/build/automationutils.py \
-  $(topsrcdir)/build/poster.zip \
   $(topsrcdir)/build/mobile/remoteautomation.py \
   $(topsrcdir)/testing/mochitest/server.js \
   $(topsrcdir)/build/pgo/server-locations.txt \
   $(NULL)
 
 $(_DEST_DIR):
 	$(NSINSTALL) -D $@
 
--- a/testing/mochitest/Makefile.in
+++ b/testing/mochitest/Makefile.in
@@ -77,17 +77,16 @@ include $(topsrcdir)/build/automation-bu
 		runtests.py \
 		automation.py \
 		runtestsremote.py \
 		runtestsvmware.py \
 		$(topsrcdir)/build/mobile/devicemanager.py \
 		$(topsrcdir)/build/mobile/devicemanagerADB.py \
 		$(topsrcdir)/build/mobile/devicemanagerSUT.py \
 		$(topsrcdir)/build/automationutils.py \
-		$(topsrcdir)/build/poster.zip \
 		$(topsrcdir)/build/mobile/remoteautomation.py \
 		gen_template.pl \
 		server.js \
 		harness-overlay.xul \
 		harness.xul \
 		browser-test-overlay.xul \
 		browser-test.js \
 		chrome-harness.js \
--- a/testing/xpcshell/Makefile.in
+++ b/testing/xpcshell/Makefile.in
@@ -60,17 +60,16 @@ TEST_HARNESS_FILES := \
   head.js \
   $(NULL)
 
 # Extra files needed from $(topsrcdir)/build
 EXTRA_BUILD_FILES := \
   automationutils.py \
   manifestparser.py \
   mozinfo.py \
-  poster.zip \
   $(NULL)
 
 # And files for running xpcshell remotely from $(topsrcdir)/build/mobile
 MOBILE_BUILD_FILES := \
   devicemanager.py \
   $(NULL)
 
 # Components / typelibs that don't get packaged with