Back out cd695cdb3b4f (bug 679759) because of test failures
authorMatt Brubeck <mbrubeck@mozilla.com>
Mon, 07 Nov 2011 13:40:42 -0800
changeset 81300 72592670532d7d4cad4c77ec26caae39b0f26a21
parent 81299 375aaf23cc2fe652e79d5b3a5a363d2963e79006
child 81301 10814cd743e3b077fe30836cfeb02d703216a989
push id434
push userclegnitto@mozilla.com
push dateWed, 21 Dec 2011 12:10:54 +0000
treeherdermozilla-beta@bddb6ed8dd47 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs679759
milestone10.0a1
backs outcd695cdb3b4fb6b09a66ee9b23fd74a425f3ff6b
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Back out cd695cdb3b4f (bug 679759) because of test failures
build/automation.py.in
build/automationutils.py
layout/tools/reftest/Makefile.in
testing/mochitest/Makefile.in
testing/xpcshell/Makefile.in
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -47,16 +47,17 @@ import re
 import select
 import shutil
 import signal
 import subprocess
 import sys
 import threading
 import tempfile
 import sqlite3
+import zipfile
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 sys.path.insert(0, SCRIPT_DIR)
 import automationutils
 
 _DEFAULT_WEB_SERVER = "127.0.0.1"
 _DEFAULT_HTTP_PORT = 8888
 _DEFAULT_SSL_PORT = 4443
@@ -92,16 +93,79 @@ else:
 # threads, which is needed to process the output of the server and application
 # processes simultaneously.
 _log = logging.getLogger()
 handler = logging.StreamHandler(sys.stdout)
 _log.setLevel(logging.INFO)
 _log.addHandler(handler)
 
 
+class ZipFileReader(object):
+  """
+  Class to read zip files in Python 2.5 and later. Limited to only what we
+  actually use.
+  """
+
+  def __init__(self, filename):
+    self._zipfile = zipfile.ZipFile(filename, "r")
+
+  def __del__(self):
+    self._zipfile.close()
+
+  def _getnormalizedpath(self, path):
+    """
+    Gets a normalized path from 'path' (or the current working directory if
+    'path' is None). Also asserts that the path exists.
+    """
+    if path is None:
+      path = os.curdir
+    path = os.path.normpath(os.path.expanduser(path))
+    assert os.path.isdir(path)
+    return path
+
+  def _extractname(self, name, path):
+    """
+    Extracts a file with the given name from the zip file to the given path.
+    Also creates any directories needed along the way.
+    """
+    filename = os.path.normpath(os.path.join(path, name))
+    if name.endswith("/"):
+      os.makedirs(filename)
+    else:
+      path = os.path.split(filename)[0]
+      if not os.path.isdir(path):
+        os.makedirs(path)
+      with open(filename, "wb") as dest:
+        dest.write(self._zipfile.read(name))
+
+  def namelist(self):
+    return self._zipfile.namelist()
+
+  def read(self, name):
+    return self._zipfile.read(name)
+
+  def extract(self, name, path = None):
+    if hasattr(self._zipfile, "extract"):
+      return self._zipfile.extract(name, path)
+
+    # This will throw if name is not part of the zip file.
+    self._zipfile.getinfo(name)
+
+    self._extractname(name, self._getnormalizedpath(path))
+
+  def extractall(self, path = None):
+    if hasattr(self._zipfile, "extractall"):
+      return self._zipfile.extractall(path)
+
+    path = self._getnormalizedpath(path)
+
+    for name in self._zipfile.namelist():
+      self._extractname(name, path)
+
+
 #################
 # PROFILE SETUP #
 #################
 
 class SyntaxError(Exception):
   "Signifies a syntax error on a particular line in server-locations.txt."
 
   def __init__(self, lineno, msg = None):
@@ -983,17 +1047,17 @@ user_pref("camino.use_system_proxy_setti
 
     installRDFFilename = "install.rdf"
 
     extensionsRootDir = os.path.join(profileDir, "extensions", "staged")
     if not os.path.isdir(extensionsRootDir):
       os.makedirs(extensionsRootDir)
 
     if os.path.isfile(extensionSource):
-      reader = automationutils.ZipFileReader(extensionSource)
+      reader = ZipFileReader(extensionSource)
 
       for filename in reader.namelist():
         # Sanity check the zip file.
         if os.path.isabs(filename):
           self.log.info("INFO | automation.py | Cannot install extension, bad files in xpi")
           return
 
         # We may need to dig the extensionID out of the zip file...
--- a/build/automationutils.py
+++ b/build/automationutils.py
@@ -31,22 +31,21 @@
 # use your version of this file under the terms of the MPL, indicate your
 # decision by deleting the provisions above and replace them with the notice
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK ***** */
 
-import glob, logging, os, platform, shutil, subprocess, sys, tempfile, urllib2, zipfile
+import glob, logging, os, platform, shutil, subprocess, sys
 import re
 from urlparse import urlparse
 
 __all__ = [
-  "ZipFileReader",
   "addCommonOptions",
   "checkForCrashes",
   "dumpLeakLog",
   "isURL",
   "processLeakLog",
   "getDebuggerInfo",
   "DEBUGGER_INFO",
   "replaceBackSlashes",
@@ -66,78 +65,16 @@ DEBUGGER_INFO = {
   # valgrind doesn't explain much about leaks unless you set the
   # '--leak-check=full' flag.
   "valgrind": {
     "interactive": False,
     "args": "--leak-check=full"
   }
 }
 
-class ZipFileReader(object):
-  """
-  Class to read zip files in Python 2.5 and later. Limited to only what we
-  actually use.
-  """
-
-  def __init__(self, filename):
-    self._zipfile = zipfile.ZipFile(filename, "r")
-
-  def __del__(self):
-    self._zipfile.close()
-
-  def _getnormalizedpath(self, path):
-    """
-    Gets a normalized path from 'path' (or the current working directory if
-    'path' is None). Also asserts that the path exists.
-    """
-    if path is None:
-      path = os.curdir
-    path = os.path.normpath(os.path.expanduser(path))
-    assert os.path.isdir(path)
-    return path
-
-  def _extractname(self, name, path):
-    """
-    Extracts a file with the given name from the zip file to the given path.
-    Also creates any directories needed along the way.
-    """
-    filename = os.path.normpath(os.path.join(path, name))
-    if name.endswith("/"):
-      os.makedirs(filename)
-    else:
-      path = os.path.split(filename)[0]
-      if not os.path.isdir(path):
-        os.makedirs(path)
-      with open(filename, "wb") as dest:
-        dest.write(self._zipfile.read(name))
-
-  def namelist(self):
-    return self._zipfile.namelist()
-
-  def read(self, name):
-    return self._zipfile.read(name)
-
-  def extract(self, name, path = None):
-    if hasattr(self._zipfile, "extract"):
-      return self._zipfile.extract(name, path)
-
-    # This will throw if name is not part of the zip file.
-    self._zipfile.getinfo(name)
-
-    self._extractname(name, self._getnormalizedpath(path))
-
-  def extractall(self, path = None):
-    if hasattr(self._zipfile, "extractall"):
-      return self._zipfile.extractall(path)
-
-    path = self._getnormalizedpath(path)
-
-    for name in self._zipfile.namelist():
-      self._extractname(name, path)
-
 log = logging.getLogger()
 
 def isURL(thing):
   """Return True if |thing| looks like a URL."""
   return urlparse(thing).scheme != ''
 
 def addCommonOptions(parser, defaults={}):
   parser.add_option("--xre-path",
@@ -160,87 +97,88 @@ def addCommonOptions(parser, defaults={}
                            "the application on the command line")
   parser.add_option("--debugger-interactive",
                     action = "store_true", dest = "debuggerInteractive",
                     help = "prevents the test harness from redirecting "
                         "stdout and stderr for interactive debuggers")
 
 def checkForCrashes(dumpDir, symbolsPath, testName=None):
   stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
+  stackwalkCGI = os.environ.get('MINIDUMP_STACKWALK_CGI', None)
   # try to get the caller's filename if no test name is given
   if testName is None:
     try:
       testName = os.path.basename(sys._getframe(1).f_code.co_filename)
     except:
       testName = "unknown"
 
-  # Check preconditions
+  foundCrash = False
   dumps = glob.glob(os.path.join(dumpDir, '*.dmp'))
-  if len(dumps) == 0:
-    return False
-
-  foundCrash = False
-  removeSymbolsPath = False
-
-  # If our symbols are at a remote URL, download them now
-  if isURL(symbolsPath):
-    print "Downloading symbols from: " + symbolsPath
-    removeSymbolsPath = True
-    # Get the symbols and write them to a temporary zipfile
-    data = urllib2.urlopen(symbolsPath)
-    symbolsFile = tempfile.TemporaryFile()
-    symbolsFile.write(data.read())
-    # extract symbols to a temporary directory (which we'll delete after
-    # processing all crashes)
-    symbolsPath = tempfile.mkdtemp()
-    zfile = ZipFileReader(symbolsFile)
-    zfile.extractall(symbolsPath)
-
-  try:
-    for d in dumps:
-      log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
-      print "Crash dump filename: " + d
-      if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
-        # run minidump stackwalk
-        p = subprocess.Popen([stackwalkPath, d, symbolsPath],
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        (out, err) = p.communicate()
-        if len(out) > 3:
-          # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
-          print out
+  for d in dumps:
+    log.info("PROCESS-CRASH | %s | application crashed (minidump found)", testName)
+    print "Crash dump filename: " + d
+    if symbolsPath and stackwalkPath and os.path.exists(stackwalkPath):
+      p = subprocess.Popen([stackwalkPath, d, symbolsPath],
+                           stdout=subprocess.PIPE,
+                           stderr=subprocess.PIPE)
+      (out, err) = p.communicate()
+      if len(out) > 3:
+        # minidump_stackwalk is chatty, so ignore stderr when it succeeds.
+        print out
+      else:
+        print "stderr from minidump_stackwalk:"
+        print err
+      if p.returncode != 0:
+        print "minidump_stackwalk exited with return code %d" % p.returncode
+    elif stackwalkCGI and symbolsPath and isURL(symbolsPath):
+      f = None
+      try:
+        f = open(d, "rb")
+        sys.path.append(os.path.join(os.path.dirname(__file__), "poster.zip"))
+        from poster.encode import multipart_encode
+        from poster.streaminghttp import register_openers
+        import urllib2
+        register_openers()
+        datagen, headers = multipart_encode({"minidump": f,
+                                             "symbols": symbolsPath})
+        request = urllib2.Request(stackwalkCGI, datagen, headers)
+        result = urllib2.urlopen(request).read()
+        if len(result) > 3:
+          print result
         else:
-          print "stderr from minidump_stackwalk:"
-          print err
-        if p.returncode != 0:
-          print "minidump_stackwalk exited with return code %d" % p.returncode
+          print "stackwalkCGI returned nothing."
+      finally:
+        if f:
+          f.close()
+    else:
+      if not symbolsPath:
+        print "No symbols path given, can't process dump."
+      if not stackwalkPath and not stackwalkCGI:
+        print "Neither MINIDUMP_STACKWALK nor MINIDUMP_STACKWALK_CGI is set, can't process dump."
       else:
-        if not symbolsPath:
-          print "No symbols path given, can't process dump."
-        if not stackwalkPath:
-          print "MINIDUMP_STACKWALK not set, can't process dump."
-        elif stackwalkPath and not os.path.exists(stackwalkPath):
+        if stackwalkPath and not os.path.exists(stackwalkPath):
           print "MINIDUMP_STACKWALK binary not found: %s" % stackwalkPath
-      dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
-      if dumpSavePath:
-        shutil.move(d, dumpSavePath)
-        print "Saved dump as %s" % os.path.join(dumpSavePath,
-                                                os.path.basename(d))
-      else:
-        os.remove(d)
-      extra = os.path.splitext(d)[0] + ".extra"
-      if os.path.exists(extra):
-        os.remove(extra)
-      foundCrash = True
-  finally:
-    if removeSymbolsPath:
-      shutil.rmtree(symbolsPath)
+        elif stackwalkCGI and not isURL(stackwalkCGI):
+          print "MINIDUMP_STACKWALK_CGI is not a URL: %s" % stackwalkCGI
+        elif symbolsPath and not isURL(symbolsPath):
+          print "symbolsPath is not a URL: %s" % symbolsPath
+    dumpSavePath = os.environ.get('MINIDUMP_SAVE_PATH', None)
+    if dumpSavePath:
+      shutil.move(d, dumpSavePath)
+      print "Saved dump as %s" % os.path.join(dumpSavePath,
+                                              os.path.basename(d))
+    else:
+      os.remove(d)
+    extra = os.path.splitext(d)[0] + ".extra"
+    if os.path.exists(extra):
+      os.remove(extra)
+    foundCrash = True
 
   return foundCrash
-
+  
 def getFullPath(directory, path):
   "Get an absolute path relative to 'directory'."
   return os.path.normpath(os.path.join(directory, os.path.expanduser(path)))
 
 def searchPath(directory, path):
   "Go one step beyond getFullPath and try the various folders in PATH"
   # Try looking in the current working directory first.
   newpath = getFullPath(directory, path)
--- a/layout/tools/reftest/Makefile.in
+++ b/layout/tools/reftest/Makefile.in
@@ -79,16 +79,17 @@ endif
 _HARNESS_FILES = \
   $(srcdir)/runreftest.py \
   $(srcdir)/remotereftest.py \
   automation.py \
   $(topsrcdir)/build/mobile/devicemanager.py \
   $(topsrcdir)/build/mobile/devicemanagerADB.py \
   $(topsrcdir)/build/mobile/devicemanagerSUT.py \
   $(topsrcdir)/build/automationutils.py \
+  $(topsrcdir)/build/poster.zip \
   $(topsrcdir)/build/mobile/remoteautomation.py \
   $(topsrcdir)/testing/mochitest/server.js \
   $(topsrcdir)/build/pgo/server-locations.txt \
   $(NULL)
 
 $(_DEST_DIR):
 	$(NSINSTALL) -D $@
 
--- a/testing/mochitest/Makefile.in
+++ b/testing/mochitest/Makefile.in
@@ -77,16 +77,17 @@ include $(topsrcdir)/build/automation-bu
 		runtests.py \
 		automation.py \
 		runtestsremote.py \
 		runtestsvmware.py \
 		$(topsrcdir)/build/mobile/devicemanager.py \
 		$(topsrcdir)/build/mobile/devicemanagerADB.py \
 		$(topsrcdir)/build/mobile/devicemanagerSUT.py \
 		$(topsrcdir)/build/automationutils.py \
+		$(topsrcdir)/build/poster.zip \
 		$(topsrcdir)/build/mobile/remoteautomation.py \
 		gen_template.pl \
 		server.js \
 		harness-overlay.xul \
 		harness.xul \
 		browser-test-overlay.xul \
 		browser-test.js \
 		chrome-harness.js \
--- a/testing/xpcshell/Makefile.in
+++ b/testing/xpcshell/Makefile.in
@@ -60,16 +60,17 @@ TEST_HARNESS_FILES := \
   head.js \
   $(NULL)
 
 # Extra files needed from $(topsrcdir)/build
 EXTRA_BUILD_FILES := \
   automationutils.py \
   manifestparser.py \
   mozinfo.py \
+  poster.zip \
   $(NULL)
 
 # And files for running xpcshell remotely from $(topsrcdir)/build/mobile
 MOBILE_BUILD_FILES := \
   devicemanager.py \
   $(NULL)
 
 # Components / typelibs that don't get packaged with