Back out cd695cdb3b4f (bug 679759) because of test failures
authorMatt Brubeck <mbrubeck@mozilla.com>
Mon, 07 Nov 2011 13:40:42 -0800
changeset 80595 72592670532d7d4cad4c77ec26caae39b0f26a21
parent 80594 375aaf23cc2fe652e79d5b3a5a363d2963e79006
child 80596 10814cd743e3b077fe30836cfeb02d703216a989
push id506
push userclegnitto@mozilla.com
push dateWed, 09 Nov 2011 02:03:18 +0000
treeherdermozilla-aurora@63587fc7bb93 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs679759
milestone10.0a1
backs outcd695cdb3b4fb6b09a66ee9b23fd74a425f3ff6b
Back out cd695cdb3b4f (bug 679759) because of test failures
build/automation.py.in
build/automationutils.py
layout/tools/reftest/Makefile.in
testing/mochitest/Makefile.in
testing/xpcshell/Makefile.in
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -47,16 +47,17 @@ import re
 import select
 import shutil
 import signal
 import subprocess
 import sys
 import threading
 import tempfile
 import sqlite3
+import zipfile
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 sys.path.insert(0, SCRIPT_DIR)
 import automationutils
 
 _DEFAULT_WEB_SERVER = "127.0.0.1"
 _DEFAULT_HTTP_PORT = 8888
 _DEFAULT_SSL_PORT = 4443
@@ -92,16 +93,79 @@ else:
 # threads, which is needed to process the output of the server and application
 # processes simultaneously.
 _log = logging.getLogger()
 handler = logging.StreamHandler(sys.stdout)
 _log.setLevel(logging.INFO)
 _log.addHandler(handler)
 
 
+class ZipFileReader(object):
+  """
+  Class to read zip files in Python 2.5 and later. Limited to only what we
+  actually use.
+  """
+
+  def __init__(self, filename):
+    self._zipfile = zipfile.ZipFile(filename, "r")
+
+  def __del__(self):
+    self._zipfile.close()
+
+  def _getnormalizedpath(self, path):
+    """
+    Gets a normalized path from 'path' (or the current working directory if
+    'path' is None). Also asserts that the path exists.
+    """
+    if path is None:
+      path = os.curdir
+    path = os.path.normpath(os.path.expanduser(path))
+    assert os.path.isdir(path)
+    return path
+
+  def _extractname(self, name, path):
+    """
+    Extracts a file with the given name from the zip file to the given path.
+    Also creates any directories needed along the way.
+    """
+    filename = os.path.normpath(os.path.join(path, name))
+    if name.endswith("/"):
+      os.makedirs(filename)
+    else:
+      path = os.path.split(filename)[0]
+      if not os.path.isdir(path):
+        os.makedirs(path)
+      with open(filename, "wb") as dest:
+        dest.write(self._zipfile.read(name))
+
+  def namelist(self):
+    return self._zipfile.namelist()
+
+  def read(self, name):
+    return self._zipfile.read(name)
+
+  def extract(self, name, path = None):
+    if hasattr(self._zipfile, "extract"):
+      return self._zipfile.extract(name, path)
+
+    # This will throw if name is not part of the zip file.
+    self._zipfile.getinfo(name)
+
+    self._extractname(name, self._getnormalizedpath(path))
+
+  def extractall(self, path = None):
+    if hasattr(self._zipfile, "extractall"):
+      return self._zipfile.extractall(path)
+
+    path = self._getnormalizedpath(path)
+
+    for name in self._zipfile.namelist():
+      self._extractname(name, path)
+
+
 #################
 # PROFILE SETUP #
 #################
 
 class SyntaxError(Exception):
   "Signifies a syntax error on a particular line in server-locations.txt."
 
   def __init__(self, lineno, msg = None):
@@ -983,17 +1047,17 @@ user_pref("camino.use_system_proxy_setti
 
     installRDFFilename = "install.rdf"
 
     extensionsRootDir = os.path.join(profileDir, "extensions", "staged")
     if not os.path.isdir(extensionsRootDir):
       os.makedirs(extensionsRootDir)
 
     if os.path.isfile(extensionSource):
-      reader = automationutils.ZipFileReader(extensionSource)
+      reader = ZipFileReader(extensionSource)
 
       for filename in reader.namelist():
         # Sanity check the zip file.
         if os.path.isabs(filename):
           self.log.info("INFO | automation.py | Cannot install extension, bad files in xpi")
           return
 
         # We may need to dig the extensionID out of the zip file...
--- a/build/automationutils.py
+++ b/build/automationutils.py
@@ -31,22 +31,21 @@
 # use your version of this file under the terms of the MPL, indicate your
 # decision by deleting the provisions above and replace them with the notice
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK ***** */
 
-import glob, logging, os, platform, shutil, subprocess, sys, tempfile, urllib2, zipfile
+import glob, logging, os, platform, shutil, subprocess, sys
 import re
 from urlparse import urlparse
 
 __all__ = [
-  "ZipFileReader",
   "addCommonOptions",
   "checkForCrashes",
   "dumpLeakLog",
   "isURL",
   "processLeakLog",
   "getDebuggerInfo",
   "DEBUGGER_INFO",
   "replaceBackSlashes",
@@ -66,78 +65,16 @@ DEBUGGER_INFO = {
   # valgrind doesn't explain much about leaks unless you set the
   # '--leak-check=full' flag.
   "valgrind": {
     "interactive": False,
     "args": "--leak-check=full"
   }
 }
 
-class ZipFileReader(object):
-  """
-  Class to read zip files in Python 2.5 and later. Limited to only what we
-  actually use.
-  """
-
-  def __init__(self, filename):
-    self._zipfile = zipfile.ZipFile(filename, "r")
-
-  def __del__(self):
-    self._zipfile.close()
-
-  def _getnormalizedpath(self, path):
-    """
-    Gets a normalized path from 'path' (or the current working directory if
-    'path' is None). Also asserts that the path exists.
-    """
-    if path is None:
-      path = os.curdir
-    path = os.path.normpath(os.path.expanduser(path))
-    assert os.path.isdir(path)
-    return path
-
-  def _extractname(self, name, path):
-    """
-    Extracts a file with the given name from the zip file to the given path.
-    Also creates any directories needed along the way.
-    """
-    filename = os.path.normpath(os.path.join(path, name))
-    if name.endswith("/"):
-      os.makedirs(filename)
-    else:
-      path = os.path.split(filename)[0]
-      if not os.path.isdir(path):
-        os.makedirs(path)
-      with open(filename, "wb") as dest:
-        dest.write(self._zipfile.read(name))
-
-  def namelist(self):
-    return self._zipfile.namelist()
-
-  def read(self, name):
-    return self._zipfile.read(name)
-
-  def extract(self, name, path = None):
-    if hasattr(self._zipfile, "extract"):
-      return self._zipfile.extract(name, path)
-
-    # This will throw if name is not part of the zip file.
-    self._zipfile.getinfo(name)
-
-    self._extractname(name, self._getnormalizedpath(path))
-
-  def extractall(self, path = None):
-    if hasattr(self._zipfile, "extractall"):
-      return self._zipfile.extractall(path)
-
-    path = self._getnormalizedpath(path)
-
-    for name in self._zipfile.namelist():
-      self._extractname(name, path)
-
 log = logging.getLogger()
 
 def isURL(thing):
   """Return True if |thing| looks like a URL."""
   return urlparse(thing).scheme != ''
 
 def addCommonOptions(parser, defaults={}):
   parser.add_option("--xre-path",
@@ -160,87 +97,88 @@ def addCommonOptions(parser, defaults={}
                            "the application on the command line")
   parser.add_option("--debugger-interactive",
                     action = "store_true", dest = "debuggerInteractive",
                     help = "prevents the test harness from redirecting "
                         "stdout and stderr for interactive debuggers")
 
 def checkForCrashes(dumpDir, symbolsPath, testName=None):
   stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
+  stackwalkCGI = os.environ.get('MINIDUMP_STACKWALK_CGI', None)
   # try to get the caller's filename if no test name is given
   if testName is None:
     try:
       testName = os.path.basename(sys._getframe(1).f_code.co_filename)
     except:
       testName = "unknown"
 
-  # Check preconditions
+  foundCrash = False
   dumps = glob.glob(os.path.join(dumpDir, '*.dmp'))
-  if len(dumps) == 0:
-    return False
-
-  foundCrash = False
-  removeSymbolsPath = False
-
-  # If our symbols are at a remote URL, download them now
-  if isURL(symbolsPath):
-    print "Downloading symbols from: " + symbolsPath
-    removeSymbolsPath = True
-    # Get the symbols and write them to a temporary zipfile
-    data = urllib2.urlopen(symbolsPath)
-    symbolsFile = tempfile.TemporaryFile()
-    symbolsFile.write(data.read())
-    # extract symbols to a temporary directory (which we'll delete after
-    # processing all crashes)
-    symbolsPath = tempfile.mkdtemp()
-    zfile = ZipFileReader(symbolsFile)
-    zfile.extractall(symbolsPath)
-
-  try:
-    for d in dumps:
-      log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
-      print "Crash dump filename: " + d
-      if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
-        # run minidump stackwalk
-        p = subprocess.Popen([stackwalkPath, d, symbolsPath],
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        (out, err) = p.communicate()
-        if len(out) > 3:
-          # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
-          print out
+  for d in dumps:
+    log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
+    print "Crash dump filename: " + d
+    if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
+      p = subprocess.Popen([stackwalkPath, d, symbolsPath],
+                           stdout=subprocess.PIPE,
+                           stderr=subprocess.PIPE)
+      (out, err) = p.communicate()
+      if len(out) > 3:
+        # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
+        print out
+      else:
+        print "stderr from minidump_stackwalk:"
+        print err
+      if p.returncode != 0:
+        print "minidump_stackwalk exited with return code %d" % p.returncode
+    elif stackwalkCGI and symbolsPath and isURL(symbolsPath):
+      f = None
+      try:
+        f = open(d, "rb")
+        sys.path.append(os.path.join(os.path.dirname(__file__), "poster.zip"))
+        from poster.encode import multipart_encode
+        from poster.streaminghttp import register_openers
+        import urllib2
+        register_openers()
+        datagen, headers = multipart_encode({"minidump": f,
+                                             "symbols": symbolsPath})
+        request = urllib2.Request(stackwalkCGI, datagen, headers)
+        result = urllib2.urlopen(request).read()
+        if len(result) > 3:
+          print result
         else:
-          print "stderr from minidump_stackwalk:"
-          print err
-        if p.returncode != 0:
-          print "minidump_stackwalk exited with return code %d" % p.returncode
+          print "stackwalkCGI returned nothing."
+      finally:
+        if f:
+          f.close()
+    else:
+      if not symbolsPath:
+        print "No symbols path given, can't process dump."
+      if not stackwalkPath and not stackwalkCGI:
+        print "Neither MINIDUMP_STACKWALK nor MINIDUMP_STACKWALK_CGI is set, can't process dump."
       else:
-        if not symbolsPath:
-          print "No symbols path given, can't process dump."
-        if not stackwalkPath:
-          print "MINIDUMP_STACKWALK not set, can't process dump."
-        elif stackwalkPath and not os.path.exists(stackwalkPath):
+        if stackwalkPath and not os.path.exists(stackwalkPath):
           print "MINIDUMP_STACKWALK binary not found: %s" % stackwalkPath
-      dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
-      if dumpSavePath:
-        shutil.move(d, dumpSavePath)
-        print "Saved dump as %s" % os.path.join(dumpSavePath,
-                                                os.path.basename(d))
-      else:
-        os.remove(d)
-      extra = os.path.splitext(d)[0] + ".extra"
-      if os.path.exists(extra):
-        os.remove(extra)
-      foundCrash = True
-  finally:
-    if removeSymbolsPath:
-      shutil.rmtree(symbolsPath)
+        elif stackwalkCGI and not isURL(stackwalkCGI):
+          print "MINIDUMP_STACKWALK_CGI is not a URL: %s" % stackwalkCGI
+        elif symbolsPath and not isURL(symbolsPath):
+          print "symbolsPath is not a URL: %s" % symbolsPath
+    dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
+    if dumpSavePath:
+      shutil.move(d, dumpSavePath)
+      print "Saved dump as %s" % os.path.join(dumpSavePath,
+                                              os.path.basename(d))
+    else:
+      os.remove(d)
+    extra = os.path.splitext(d)[0] + ".extra"
+    if os.path.exists(extra):
+      os.remove(extra)
+    foundCrash = True
 
   return foundCrash
-
+  
 def getFullPath(directory, path):
   "Get an absolute path relative to 'directory'."
   return os.path.normpath(os.path.join(directory, os.path.expanduser(path)))
 
 def searchPath(directory, path):
   "Go one step beyond getFullPath and try the various folders in PATH"
   # Try looking in the current working directory first.
   newpath = getFullPath(directory, path)
--- a/layout/tools/reftest/Makefile.in
+++ b/layout/tools/reftest/Makefile.in
@@ -79,16 +79,17 @@ endif
 _HARNESS_FILES = \
   $(srcdir)/runreftest.py \
   $(srcdir)/remotereftest.py \
   automation.py \
   $(topsrcdir)/build/mobile/devicemanager.py \
   $(topsrcdir)/build/mobile/devicemanagerADB.py \
   $(topsrcdir)/build/mobile/devicemanagerSUT.py \
   $(topsrcdir)/build/automationutils.py \
+  $(topsrcdir)/build/poster.zip \
   $(topsrcdir)/build/mobile/remoteautomation.py \
   $(topsrcdir)/testing/mochitest/server.js \
   $(topsrcdir)/build/pgo/server-locations.txt \
   $(NULL)
 
 $(_DEST_DIR):
 	$(NSINSTALL) -D $@
 
--- a/testing/mochitest/Makefile.in
+++ b/testing/mochitest/Makefile.in
@@ -77,16 +77,17 @@ include $(topsrcdir)/build/automation-bu
 		runtests.py \
 		automation.py \
 		runtestsremote.py \
 		runtestsvmware.py \
 		$(topsrcdir)/build/mobile/devicemanager.py \
 		$(topsrcdir)/build/mobile/devicemanagerADB.py \
 		$(topsrcdir)/build/mobile/devicemanagerSUT.py \
 		$(topsrcdir)/build/automationutils.py \
+		$(topsrcdir)/build/poster.zip \
 		$(topsrcdir)/build/mobile/remoteautomation.py \
 		gen_template.pl \
 		server.js \
 		harness-overlay.xul \
 		harness.xul \
 		browser-test-overlay.xul \
 		browser-test.js \
 		chrome-harness.js \
--- a/testing/xpcshell/Makefile.in
+++ b/testing/xpcshell/Makefile.in
@@ -60,16 +60,17 @@ TEST_HARNESS_FILES := \
   head.js \
   $(NULL)
 
 # Extra files needed from $(topsrcdir)/build
 EXTRA_BUILD_FILES := \
   automationutils.py \
   manifestparser.py \
   mozinfo.py \
+  poster.zip \
   $(NULL)
 
 # And files for running xpcshell remotely from $(topsrcdir)/build/mobile
 MOBILE_BUILD_FILES := \
   devicemanager.py \
   $(NULL)
 
 # Components / typelibs that don't get packaged with