Bug 498128 - xpcshell-tests: add option to enable/disable file logging; (Av3a) Support --[no-]logfiles in xpcst suite, improve dir/file handling in 3 suites
authorSerge Gautherie <sgautherie.bz@free.fr>
Mon, 21 Sep 2009 18:19:21 +0200
changeset 32920 72ae5f35e8fa89deb67093b6023c0a15b4c6ddb7
parent 32919 6ae554d40a781e703980277aa0c40b072b68d4e8
child 32921 bf607bae5e3896b2702c9f0be3c3aaa77089c638
push idunknown
push userunknown
push dateunknown
bugs498128
milestone1.9.3a1pre
Bug 498128 - xpcshell-tests: add option to enable/disable file logging; (Av3a) Support --[no-]logfiles in xpcst suite, improve dir/file handling in 3 suites r=ted.mielczarek
layout/tools/reftest/runreftest.py
testing/mochitest/runtests.py.in
testing/testsuite-targets.mk
testing/xpcshell/runxpcshelltests.py
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -52,28 +52,25 @@ from tempfile import mkdtemp
 oldcwd = os.getcwd()
 os.chdir(SCRIPT_DIRECTORY)
 
 def getFullPath(path):
   "Get an absolute path relative to oldcwd."
   return os.path.normpath(os.path.join(oldcwd, os.path.expanduser(path)))
 
 def createReftestProfile(options, profileDir):
-  "Sets up a clean profile for reftest."
+  "Sets up a profile for reftest."
 
-  # Start with a clean slate.
-  shutil.rmtree(profileDir, True)
-  os.mkdir(profileDir)
-  # reftest should only need the dump pref set
+  # Set preferences.
   prefsFile = open(os.path.join(profileDir, "user.js"), "w")
   prefsFile.write("""user_pref("browser.dom.window.dump.enabled", true);
 """)
   prefsFile.write('user_pref("reftest.timeout", %d);' % options.timeout)
+  prefsFile.close()
 
-  prefsFile.close()
   # install the reftest extension bits into the profile
   profileExtensionsPath = os.path.join(profileDir, "extensions")
   os.mkdir(profileExtensionsPath)
   reftestExtensionPath = os.path.join(SCRIPT_DIRECTORY, "reftest")
   extFile = open(os.path.join(profileExtensionsPath, "reftest@mozilla.org"), "w")
   extFile.write(reftestExtensionPath)
   extFile.close()
 
@@ -165,17 +162,17 @@ Are you executing $objdir/_tests/reftest
     reftestlist = getFullPath(args[0])
     status = automation.runApp(None, browserEnv, options.app, profileDir,
                                ["-reftest", reftestlist],
                                xrePath=options.xrePath,
                                symbolsPath=options.symbolsPath)
     processLeakLog(leakLogFile, options.leakThreshold)
     automation.log.info("\nREFTEST INFO | runreftest.py | Running tests: end.")
   finally:
-    if profileDir is not None:
+    if profileDir:
       shutil.rmtree(profileDir)
   sys.exit(status)
 
 def copyExtraFilesToProfile(options, profileDir):
   "Copy extra files or dirs specified on the command line to the testing profile."
   for f in options.extraProfileFiles:
     abspath = getFullPath(f)
     dest = os.path.join(profileDir, os.path.basename(abspath))
--- a/testing/mochitest/runtests.py.in
+++ b/testing/mochitest/runtests.py.in
@@ -80,17 +80,17 @@ else:
     SERVER_STARTUP_TIMEOUT = 45
 
 oldcwd = os.getcwd()
 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 os.chdir(SCRIPT_DIRECTORY)
 
 PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile")
 
-LEAK_REPORT_FILE = PROFILE_DIRECTORY + "/" + "leaks-report.log"
+LEAK_REPORT_FILE = os.path.join(PROFILE_DIRECTORY, "runtests_leaks.log")
 
 # Map of debugging programs to information about them, like default arguments
 # and whether or not they are interactive.
 DEBUGGER_INFO = {
   # gdb requires that you supply the '--args' flag in order to pass arguments
   # after the executable name to the executable.
   "gdb": {
     "interactive": True,
@@ -416,17 +416,16 @@ Are you executing $objdir/_tests/testing
   server.start()
 
   # If we're lucky, the server has fully started by now, and all paths are
   # ready, etc.  However, xpcshell cold start times suck, at least for debug
   # builds.  We'll try to connect to the server for awhile, and if we fail,
   # we'll try to kill the server and exit with an error.
   server.ensureReady(SERVER_STARTUP_TIMEOUT)
 
-
   # URL parameters to test URL:
   #
   # autorun -- kick off tests automatically
   # closeWhenDone -- runs quit.js after tests
   # logFile -- logs test run to an absolute path
   #
   
   # consoleLevel, fileLevel: set the logging level of the console and
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -96,16 +96,17 @@ crashtest:
 # Execute all xpcshell tests in the directories listed in the manifest.
 # See also config/rules.mk 'xpcshell-tests' target for local execution.
 # Usage: |make [TEST_PATH=...] [EXTRA_TEST_ARGS=...] xpcshell-tests|.
 xpcshell-tests:
 	$(PYTHON) -u $(topsrcdir)/config/pythonpath.py \
 	  -I$(topsrcdir)/build \
 	  $(topsrcdir)/testing/xpcshell/runxpcshelltests.py \
 	  --manifest=$(DEPTH)/_tests/xpcshell/all-test-dirs.list \
+	  --no-logfiles \
 	  --symbols-path=$(DIST)/crashreporter-symbols \
 	  $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS) \
 	  $(DIST)/bin/xpcshell
 
 
 # Package up the tests and test harnesses
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
--- a/testing/xpcshell/runxpcshelltests.py
+++ b/testing/xpcshell/runxpcshelltests.py
@@ -33,17 +33,16 @@
 # decision by deleting the provisions above and replace them with the notice
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK ***** */
 
 import re, sys, os, os.path, logging, shutil
-import tempfile
 from glob import glob
 from optparse import OptionParser
 from subprocess import Popen, PIPE, STDOUT
 from tempfile import mkdtemp
 
 from automationutils import addCommonOptions, checkForCrashes, dumpLeakLog
 
 # Init logging
@@ -64,28 +63,34 @@ def readManifest(manifest):
       path = os.path.join(manifestdir, dir)
       if os.path.isdir(path):
         testdirs.append(path)
     f.close()
   except:
     pass # just eat exceptions
   return testdirs
 
-def runTests(xpcshell, testdirs=[], xrePath=None, testPath=None,
-             manifest=None, interactive=False, symbolsPath=None):
-  """Run the tests in |testdirs| using the |xpcshell| executable.
+def runTests(xpcshell, xrePath=None, symbolsPath=None,
+             manifest=None, testdirs=[], testPath=None,
+             interactive=False, logfiles=True):
+  """Run xpcshell tests.
 
+  |xpcshell|, is the xpcshell executable to use to run the tests.
   |xrePath|, if provided, is the path to the XRE to use.
-  |testPath|, if provided, indicates a single path and/or test to run.
+  |symbolsPath|, if provided is the path to a directory containing
+    breakpad symbols for processing crashes in tests.
   |manifest|, if provided, is a file containing a list of
     test directories to run.
+  |testdirs|, if provided, is a list of absolute paths of test directories.
+    No-manifest only option.
+  |testPath|, if provided, indicates a single path and/or test to run.
   |interactive|, if set to True, indicates to provide an xpcshell prompt
     instead of automatically executing the test.
-  |symbolsPath|, if provided is the path to a directory containing
-    breakpad symbols for processing crashes in tests.
+  |logfiles|, if set to False, indicates not to save output to log files.
+    Non-interactive only option.
   """
 
   if not testdirs and not manifest:
     # nothing to test!
     print >>sys.stderr, "Error: No test dirs or test manifest specified!"
     return False
 
   passCount = 0
@@ -97,21 +102,16 @@ def runTests(xpcshell, testdirs=[], xreP
   httpdJSPath = os.path.join(os.path.dirname(xpcshell), "components", "httpd.js").replace("\\", "/");
 
   env = dict(os.environ)
   # Make assertions fatal
   env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
   # Don't launch the crash reporter client
   env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 
-  # Enable leaks (only) detection to its own log file.
-  # Each test will overwrite it.
-  leakLogFile = os.path.join(tempfile.gettempdir(), "runxpcshelltests_leaks.log")
-  env["XPCOM_MEM_LEAK_LOG"] = leakLogFile
-
   if xrePath is None:
     xrePath = os.path.dirname(xpcshell)
   else:
     xrePath = os.path.abspath(xrePath)
   if sys.platform == 'win32':
     env["PATH"] = env["PATH"] + ";" + xrePath
   elif sys.platform in ('os2emx', 'os2knix'):
     os.environ["BEGINLIBPATH"] = xrePath + ";" + env["BEGINLIBPATH"]
@@ -127,17 +127,17 @@ def runTests(xpcshell, testdirs=[], xreP
     xpcsRunArgs = [
       '-e', 'print("To start the test, type |_execute_test();|.");',
       '-i']
     pStdout = None
     pStderr = None
   else:
     xpcsRunArgs = ['-e', '_execute_test();']
     if sys.platform == 'os2emx':
-      pStdout = None 
+      pStdout = None
     else:
       pStdout = PIPE
     pStderr = STDOUT
 
   # <head.js> has to be loaded by xpchell: it can't load itself.
   xpcsCmd = [xpcshell, '-g', xrePath, '-j', '-s'] + \
             ['-e', 'const _HTTPD_JS_PATH = "%s";' % httpdJSPath,
              '-f', os.path.join(testharnessdir, 'head.js')]
@@ -158,16 +158,17 @@ def runTests(xpcshell, testdirs=[], xreP
         testPath = testPath.rsplit('/', 1)
         singleFile = testPath[1]
         testPath = testPath[0]
     else:
       # Path only.
       # Simply remove optional ending separator.
       testPath = testPath.rstrip("/")
 
+  # Override testdirs.
   if manifest is not None:
     testdirs = readManifest(os.path.abspath(manifest))
 
   # Process each test directory individually.
   for testdir in testdirs:
     if testPath and not testdir.endswith(testPath):
       continue
 
@@ -201,101 +202,112 @@ def runTests(xpcshell, testdirs=[], xreP
            ['-e', 'const _HEAD_FILES = [%s];' % cmdH] + \
            ['-e', 'const _TAIL_FILES = [%s];' % cmdT]
 
     # Now execute each test individually.
     for test in testfiles:
       # The test file will have to be loaded after the head files.
       cmdT = ['-e', 'const _TEST_FILE = ["%s"];' %
                       os.path.join(testdir, test).replace('\\', '/')]
-      # create a temp dir that the JS harness can stick a profile in
-      profd = mkdtemp()
-      env["XPCSHELL_TEST_PROFILE_DIR"] = profd
 
-      proc = Popen(cmdH + cmdT + xpcsRunArgs,
-                   stdout=pStdout, stderr=pStderr, env=env, cwd=testdir)
-      # |stderr == None| as |pStderr| was either |None| or redirected to |stdout|.
-      stdout, stderr = proc.communicate()
+      # create a temp dir that the JS harness can stick a profile in
+      profileDir = None
+      try:
+        profileDir = mkdtemp()
+        env["XPCSHELL_TEST_PROFILE_DIR"] = profileDir
+
+        # Enable leaks (only) detection to its own log file.
+        leakLogFile = os.path.join(profileDir, "runxpcshelltests_leaks.log")
+        env["XPCOM_MEM_LEAK_LOG"] = leakLogFile
 
-      shutil.rmtree(profd, True)
+        proc = Popen(cmdH + cmdT + xpcsRunArgs,
+                     stdout=pStdout, stderr=pStderr, env=env, cwd=testdir)
+        # |stderr == None| as |pStderr| was either |None| or redirected to |stdout|.
+        stdout, stderr = proc.communicate()
 
-      if interactive:
-        # not sure what else to do here...
-        return True
+        if interactive:
+          # Not sure what else to do here...
+          return True
 
-      if proc.returncode != 0 or (stdout is not None and re.search("^TEST-UNEXPECTED-FAIL", stdout, re.MULTILINE)):
-        print """TEST-UNEXPECTED-FAIL | %s | test failed (with xpcshell return code: %d), see following log:
+        if proc.returncode != 0 or (stdout and re.search("^TEST-UNEXPECTED-FAIL", stdout, re.MULTILINE)):
+          print """TEST-UNEXPECTED-FAIL | %s | test failed (with xpcshell return code: %d), see following log:
   >>>>>>>
   %s
   <<<<<<<""" % (test, proc.returncode, stdout)
-        checkForCrashes(testdir, symbolsPath, testName=test)
-        failCount += 1
-      else:
-        print "TEST-PASS | %s | test passed" % test
-        passCount += 1
+          checkForCrashes(testdir, symbolsPath, testName=test)
+          failCount += 1
+        else:
+          print "TEST-PASS | %s | test passed" % test
+          passCount += 1
 
-      dumpLeakLog(leakLogFile, True)
-
-      if stdout is not None:
-        try:
-          f = open(test + '.log', 'w')
-          f.write(stdout)
+        dumpLeakLog(leakLogFile, True)
 
-          if os.path.exists(leakLogFile):
-            leaks = open(leakLogFile, "r")
-            f.write(leaks.read())
-            leaks.close()
-        finally:
-          if f:
-            f.close()
+        if logfiles and stdout:
+          try:
+            f = open(test + ".log", "w")
+            f.write(stdout)
 
-      # Remove the leak detection file (here) so it can't "leak" to the next test.
-      # The file is not there if leak logging was not enabled in the xpcshell build.
-      if os.path.exists(leakLogFile):
-        os.remove(leakLogFile)
+            if os.path.exists(leakLogFile):
+              leaks = open(leakLogFile, "r")
+              f.write(leaks.read())
+              leaks.close()
+          finally:
+            if f:
+              f.close()
+      finally:
+        if profileDir:
+          shutil.rmtree(profileDir)
 
   if passCount == 0 and failCount == 0:
     print "TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?"
     failCount = 1
 
   print """INFO | Result summary:
 INFO | Passed: %d
 INFO | Failed: %d""" % (passCount, failCount)
 
   return failCount == 0
 
 def main():
   """Process command line arguments and call runTests() to do the real work."""
   parser = OptionParser()
 
   addCommonOptions(parser)
-  parser.add_option("--test-path",
-                    action="store", type="string", dest="testPath",
-                    default=None, help="single path and/or test filename to test")
   parser.add_option("--interactive",
                     action="store_true", dest="interactive", default=False,
                     help="don't automatically run tests, drop to an xpcshell prompt")
+  parser.add_option("--logfiles",
+                    action="store_true", dest="logfiles", default=True,
+                    help="create log files (default, only used to override --no-logfiles)")
   parser.add_option("--manifest",
-                    action="store", type="string", dest="manifest",
-                    default=None, help="Manifest of test directories to use")
+                    type="string", dest="manifest", default=None,
+                    help="Manifest of test directories to use")
+  parser.add_option("--no-logfiles",
+                    action="store_false", dest="logfiles",
+                    help="don't create log files")
+  parser.add_option("--test-path",
+                    type="string", dest="testPath", default=None,
+                    help="single path and/or test filename to test")
   options, args = parser.parse_args()
 
   if len(args) < 2 and options.manifest is None or \
      (len(args) < 1 and options.manifest is not None):
     print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
   or: %s --manifest=test.manifest <path to xpcshell>""" % (sys.argv[0],
                                                            sys.argv[0])
     sys.exit(1)
 
   if options.interactive and not options.testPath:
     print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
     sys.exit(1)
 
-  if not runTests(args[0], testdirs=args[1:],
+  if not runTests(args[0],
                   xrePath=options.xrePath,
+                  symbolsPath=options.symbolsPath,
+                  manifest=options.manifest,
+                  testdirs=args[1:],
                   testPath=options.testPath,
                   interactive=options.interactive,
-                  manifest=options.manifest,
-                  symbolsPath=options.symbolsPath):
+                  logfiles=options.logfiles):
     sys.exit(1)
 
 if __name__ == '__main__':
   main()