Bug 679759 - Drop MINIDUMP_STACKWALK_CGI support, let harness download symbols as needed; r=ted
authorWilliam Lachance <wlach@mozilla.com>
Tue, 15 Nov 2011 04:33:21 +0000
changeset 81906 6c596531c7942f5c81b70e8b504b43cb6e821a30
parent 81905 a2cb8a2bcff877cf7f133c22c6cfb2afc002c0f4
child 81907 6235e0c97c3baaad46453aa88997d63ac8458151
push id519
push userakeybl@mozilla.com
push dateWed, 01 Feb 2012 00:38:35 +0000
treeherdermozilla-beta@788ea1ef610b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs679759
milestone11.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 679759 - Drop MINIDUMP_STACKWALK_CGI support, let harness download symbols as needed; r=ted
build/automation.py.in
build/automationutils.py
build/poster.zip
layout/tools/reftest/Makefile.in
testing/mochitest/Makefile.in
testing/xpcshell/Makefile.in
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -47,17 +47,16 @@ import re
 import select
 import shutil
 import signal
 import subprocess
 import sys
 import threading
 import tempfile
 import sqlite3
-import zipfile
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 sys.path.insert(0, SCRIPT_DIR)
 import automationutils
 
 _DEFAULT_WEB_SERVER = "127.0.0.1"
 _DEFAULT_HTTP_PORT = 8888
 _DEFAULT_SSL_PORT = 4443
@@ -93,79 +92,16 @@ else:
 # threads, which is needed to process the output of the server and application
 # processes simultaneously.
 _log = logging.getLogger()
 handler = logging.StreamHandler(sys.stdout)
 _log.setLevel(logging.INFO)
 _log.addHandler(handler)
 
 
-class ZipFileReader(object):
-  """
-  Class to read zip files in Python 2.5 and later. Limited to only what we
-  actually use.
-  """
-
-  def __init__(self, filename):
-    self._zipfile = zipfile.ZipFile(filename, "r")
-
-  def __del__(self):
-    self._zipfile.close()
-
-  def _getnormalizedpath(self, path):
-    """
-    Gets a normalized path from 'path' (or the current working directory if
-    'path' is None). Also asserts that the path exists.
-    """
-    if path is None:
-      path = os.curdir
-    path = os.path.normpath(os.path.expanduser(path))
-    assert os.path.isdir(path)
-    return path
-
-  def _extractname(self, name, path):
-    """
-    Extracts a file with the given name from the zip file to the given path.
-    Also creates any directories needed along the way.
-    """
-    filename = os.path.normpath(os.path.join(path, name))
-    if name.endswith("/"):
-      os.makedirs(filename)
-    else:
-      path = os.path.split(filename)[0]
-      if not os.path.isdir(path):
-        os.makedirs(path)
-      with open(filename, "wb") as dest:
-        dest.write(self._zipfile.read(name))
-
-  def namelist(self):
-    return self._zipfile.namelist()
-
-  def read(self, name):
-    return self._zipfile.read(name)
-
-  def extract(self, name, path = None):
-    if hasattr(self._zipfile, "extract"):
-      return self._zipfile.extract(name, path)
-
-    # This will throw if name is not part of the zip file.
-    self._zipfile.getinfo(name)
-
-    self._extractname(name, self._getnormalizedpath(path))
-
-  def extractall(self, path = None):
-    if hasattr(self._zipfile, "extractall"):
-      return self._zipfile.extractall(path)
-
-    path = self._getnormalizedpath(path)
-
-    for name in self._zipfile.namelist():
-      self._extractname(name, path)
-
-
 #################
 # PROFILE SETUP #
 #################
 
 class SyntaxError(Exception):
   "Signifies a syntax error on a particular line in server-locations.txt."
 
   def __init__(self, lineno, msg = None):
@@ -1051,17 +987,17 @@ user_pref("camino.use_system_proxy_setti
 
     installRDFFilename = "install.rdf"
 
     extensionsRootDir = os.path.join(profileDir, "extensions", "staged")
     if not os.path.isdir(extensionsRootDir):
       os.makedirs(extensionsRootDir)
 
     if os.path.isfile(extensionSource):
-      reader = ZipFileReader(extensionSource)
+      reader = automationutils.ZipFileReader(extensionSource)
 
       for filename in reader.namelist():
         # Sanity check the zip file.
         if os.path.isabs(filename):
           self.log.info("INFO | automation.py | Cannot install extension, bad files in xpi")
           return
 
         # We may need to dig the extensionID out of the zip file...
--- a/build/automationutils.py
+++ b/build/automationutils.py
@@ -31,21 +31,23 @@
 # use your version of this file under the terms of the MPL, indicate your
 # decision by deleting the provisions above and replace them with the notice
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK ***** */
 
-import glob, logging, os, platform, shutil, subprocess, sys
+from __future__ import with_statement
+import glob, logging, os, platform, shutil, subprocess, sys, tempfile, urllib2, zipfile
 import re
 from urlparse import urlparse
 
 __all__ = [
+  "ZipFileReader",
   "addCommonOptions",
   "checkForCrashes",
   "dumpLeakLog",
   "isURL",
   "processLeakLog",
   "getDebuggerInfo",
   "DEBUGGER_INFO",
   "replaceBackSlashes",
@@ -65,16 +67,78 @@ DEBUGGER_INFO = {
   # valgrind doesn't explain much about leaks unless you set the
   # '--leak-check=full' flag.
   "valgrind": {
     "interactive": False,
     "args": "--leak-check=full"
   }
 }
 
+class ZipFileReader(object):
+  """
+  Class to read zip files in Python 2.5 and later. Limited to only what we
+  actually use.
+  """
+
+  def __init__(self, filename):
+    self._zipfile = zipfile.ZipFile(filename, "r")
+
+  def __del__(self):
+    self._zipfile.close()
+
+  def _getnormalizedpath(self, path):
+    """
+    Gets a normalized path from 'path' (or the current working directory if
+    'path' is None). Also asserts that the path exists.
+    """
+    if path is None:
+      path = os.curdir
+    path = os.path.normpath(os.path.expanduser(path))
+    assert os.path.isdir(path)
+    return path
+
+  def _extractname(self, name, path):
+    """
+    Extracts a file with the given name from the zip file to the given path.
+    Also creates any directories needed along the way.
+    """
+    filename = os.path.normpath(os.path.join(path, name))
+    if name.endswith("/"):
+      os.makedirs(filename)
+    else:
+      path = os.path.split(filename)[0]
+      if not os.path.isdir(path):
+        os.makedirs(path)
+      with open(filename, "wb") as dest:
+        dest.write(self._zipfile.read(name))
+
+  def namelist(self):
+    return self._zipfile.namelist()
+
+  def read(self, name):
+    return self._zipfile.read(name)
+
+  def extract(self, name, path = None):
+    if hasattr(self._zipfile, "extract"):
+      return self._zipfile.extract(name, path)
+
+    # This will throw if name is not part of the zip file.
+    self._zipfile.getinfo(name)
+
+    self._extractname(name, self._getnormalizedpath(path))
+
+  def extractall(self, path = None):
+    if hasattr(self._zipfile, "extractall"):
+      return self._zipfile.extractall(path)
+
+    path = self._getnormalizedpath(path)
+
+    for name in self._zipfile.namelist():
+      self._extractname(name, path)
+
 log = logging.getLogger()
 
 def isURL(thing):
   """Return True if |thing| looks like a URL."""
   return urlparse(thing).scheme != ''
 
 def addCommonOptions(parser, defaults={}):
   parser.add_option("--xre-path",
@@ -97,88 +161,87 @@ def addCommonOptions(parser, defaults={}
                            "the application on the command line")
   parser.add_option("--debugger-interactive",
                     action = "store_true", dest = "debuggerInteractive",
                     help = "prevents the test harness from redirecting "
                         "stdout and stderr for interactive debuggers")
 
 def checkForCrashes(dumpDir, symbolsPath, testName=None):
   stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
-  stackwalkCGI = os.environ.get('MINIDUMP_STACKWALK_CGI', None)
   # try to get the caller's filename if no test name is given
   if testName is None:
     try:
       testName = os.path.basename(sys._getframe(1).f_code.co_filename)
     except:
       testName = "unknown"
 
-  foundCrash = False
+  # Check preconditions
   dumps = glob.glob(os.path.join(dumpDir, '*.dmp'))
-  for d in dumps:
-    log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
-    print "Crash dump filename: " + d
-    if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
-      p = subprocess.Popen([stackwalkPath, d, symbolsPath],
-                           stdout=subprocess.PIPE,
-                           stderr=subprocess.PIPE)
-      (out, err) = p.communicate()
-      if len(out) > 3:
-        # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
-        print out
+  if len(dumps) == 0:
+    return False
+
+  foundCrash = False
+  removeSymbolsPath = False
+
+  # If our symbols are at a remote URL, download them now
+  if isURL(symbolsPath):
+    print "Downloading symbols from: " + symbolsPath
+    removeSymbolsPath = True
+    # Get the symbols and write them to a temporary zipfile
+    data = urllib2.urlopen(symbolsPath)
+    symbolsFile = tempfile.TemporaryFile()
+    symbolsFile.write(data.read())
+    # extract symbols to a temporary directory (which we'll delete after
+    # processing all crashes)
+    symbolsPath = tempfile.mkdtemp()
+    zfile = ZipFileReader(symbolsFile)
+    zfile.extractall(symbolsPath)
+
+  try:
+    for d in dumps:
+      log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
+      print "Crash dump filename: " + d
+      if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
+        # run minidump stackwalk
+        p = subprocess.Popen([stackwalkPath, d, symbolsPath],
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+        (out, err) = p.communicate()
+        if len(out) > 3:
+          # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
+          print out
+        else:
+          print "stderr from minidump_stackwalk:"
+          print err
+        if p.returncode != 0:
+          print "minidump_stackwalk exited with return code %d" % p.returncode
       else:
-        print "stderr from minidump_stackwalk:"
-        print err
-      if p.returncode != 0:
-        print "minidump_stackwalk exited with return code %d" % p.returncode
-    elif stackwalkCGI and symbolsPath and isURL(symbolsPath):
-      f = None
-      try:
-        f = open(d, "rb")
-        sys.path.append(os.path.join(os.path.dirname(__file__), "poster.zip"))
-        from poster.encode import multipart_encode
-        from poster.streaminghttp import register_openers
-        import urllib2
-        register_openers()
-        datagen, headers = multipart_encode({"minidump": f,
-                                             "symbols": symbolsPath})
-        request = urllib2.Request(stackwalkCGI, datagen, headers)
-        result = urllib2.urlopen(request).read()
-        if len(result) > 3:
-          print result
-        else:
-          print "stackwalkCGI returned nothing."
-      finally:
-        if f:
-          f.close()
-    else:
-      if not symbolsPath:
-        print "No symbols path given, can't process dump."
-      if not stackwalkPath and not stackwalkCGI:
-        print "Neither MINIDUMP_STACKWALK nor MINIDUMP_STACKWALK_CGI is set, can't process dump."
+        if not symbolsPath:
+          print "No symbols path given, can't process dump."
+        if not stackwalkPath:
+          print "MINIDUMP_STACKWALK not set, can't process dump."
+        elif stackwalkPath and not os.path.exists(stackwalkPath):
+          print "MINIDUMP_STACKWALK binary not found: %s" % stackwalkPath
+      dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
+      if dumpSavePath:
+        shutil.move(d, dumpSavePath)
+        print "Saved dump as %s" % os.path.join(dumpSavePath,
+                                                os.path.basename(d))
       else:
-        if stackwalkPath and not os.path.exists(stackwalkPath):
-          print "MINIDUMP_STACKWALK binary not found: %s" % stackwalkPath
-        elif stackwalkCGI and not isURL(stackwalkCGI):
-          print "MINIDUMP_STACKWALK_CGI is not a URL: %s" % stackwalkCGI
-        elif symbolsPath and not isURL(symbolsPath):
-          print "symbolsPath is not a URL: %s" % symbolsPath
-    dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
-    if dumpSavePath:
-      shutil.move(d, dumpSavePath)
-      print "Saved dump as %s" % os.path.join(dumpSavePath,
-                                              os.path.basename(d))
-    else:
-      os.remove(d)
-    extra = os.path.splitext(d)[0] + ".extra"
-    if os.path.exists(extra):
-      os.remove(extra)
-    foundCrash = True
+        os.remove(d)
+      extra = os.path.splitext(d)[0] + ".extra"
+      if os.path.exists(extra):
+        os.remove(extra)
+      foundCrash = True
+  finally:
+    if removeSymbolsPath:
+      shutil.rmtree(symbolsPath)
 
   return foundCrash
-  
+
 def getFullPath(directory, path):
   "Get an absolute path relative to 'directory'."
   return os.path.normpath(os.path.join(directory, os.path.expanduser(path)))
 
 def searchPath(directory, path):
   "Go one step beyond getFullPath and try the various folders in PATH"
   # Try looking in the current working directory first.
   newpath = getFullPath(directory, path)
deleted file mode 100644
index 122b4acd4698a52d208293a904130c5310851ab9..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
literal 0
Hc$@<O00001
--- a/layout/tools/reftest/Makefile.in
+++ b/layout/tools/reftest/Makefile.in
@@ -79,17 +79,16 @@ endif
 _HARNESS_FILES = \
   $(srcdir)/runreftest.py \
   $(srcdir)/remotereftest.py \
   automation.py \
   $(topsrcdir)/build/mobile/devicemanager.py \
   $(topsrcdir)/build/mobile/devicemanagerADB.py \
   $(topsrcdir)/build/mobile/devicemanagerSUT.py \
   $(topsrcdir)/build/automationutils.py \
-  $(topsrcdir)/build/poster.zip \
   $(topsrcdir)/build/mobile/remoteautomation.py \
   $(topsrcdir)/testing/mochitest/server.js \
   $(topsrcdir)/build/pgo/server-locations.txt \
   $(NULL)
 
 $(_DEST_DIR):
 	$(NSINSTALL) -D $@
 
--- a/testing/mochitest/Makefile.in
+++ b/testing/mochitest/Makefile.in
@@ -77,17 +77,16 @@ include $(topsrcdir)/build/automation-bu
 		runtests.py \
 		automation.py \
 		runtestsremote.py \
 		runtestsvmware.py \
 		$(topsrcdir)/build/mobile/devicemanager.py \
 		$(topsrcdir)/build/mobile/devicemanagerADB.py \
 		$(topsrcdir)/build/mobile/devicemanagerSUT.py \
 		$(topsrcdir)/build/automationutils.py \
-		$(topsrcdir)/build/poster.zip \
 		$(topsrcdir)/build/mobile/remoteautomation.py \
 		gen_template.pl \
 		server.js \
 		harness-overlay.xul \
 		harness.xul \
 		browser-test-overlay.xul \
 		browser-test.js \
 		chrome-harness.js \
--- a/testing/xpcshell/Makefile.in
+++ b/testing/xpcshell/Makefile.in
@@ -60,17 +60,16 @@ TEST_HARNESS_FILES := \
   head.js \
   $(NULL)
 
 # Extra files needed from $(topsrcdir)/build
 EXTRA_BUILD_FILES := \
   automationutils.py \
   manifestparser.py \
   mozinfo.py \
-  poster.zip \
   $(NULL)
 
 # And files for running xpcshell remotely from $(topsrcdir)/build/mobile
 MOBILE_BUILD_FILES := \
   devicemanager.py \
   $(NULL)
 
 # Components / typelibs that don't get packaged with