bug 468913 - Need a Makefile target to run reftest. r=dbaron,bsmedberg,jwalden (testing only change)
authorTed Mielczarek <ted.mielczarek@gmail.com>
Thu, 12 Feb 2009 08:49:53 -0500
changeset 23703 de81de26738b567c287874252912a6877edcc817
parent 23702 d06c533a8d5d0b4443b6adce12a51a01828cb49b
child 23704 c4a1a1d2e56a9b7279a2af1739058341ba49f383
push id865
push usertmielczarek@mozilla.com
push dateMon, 09 Mar 2009 10:43:15 +0000
reviewersdbaron, bsmedberg, jwalden
bugs468913
milestone1.9.1b4pre
bug 468913 - Need a Makefile target to run reftest. r=dbaron,bsmedberg,jwalden (testing only change)
build/automation-build.mk
build/automation.py.in
layout/tools/reftest/Makefile.in
layout/tools/reftest/install.rdf
layout/tools/reftest/runreftest.py
testing/mochitest/runtests.py.in
testing/testsuite-targets.mk
--- a/build/automation-build.mk
+++ b/build/automation-build.mk
@@ -7,17 +7,17 @@ endif
 
 TARGET_DIST = $(TARGET_DEPTH)/dist
 
 ifeq ($(MOZ_BUILD_APP),camino)
 browser_path = \"$(TARGET_DIST)/Camino.app/Contents/MacOS/Camino\"
 else
 ifeq ($(OS_ARCH),Darwin)
 ifdef MOZ_DEBUG
-browser_path = \"($(TARGET_DIST)/$(MOZ_APP_DISPLAYNAME)Debug.app/Contents/MacOS/$(PROGRAM)\"
+browser_path = \"$(TARGET_DIST)/$(MOZ_APP_DISPLAYNAME)Debug.app/Contents/MacOS/$(PROGRAM)\"
 else
 browser_path = \"$(TARGET_DIST)/$(MOZ_APP_DISPLAYNAME).app/Contents/MacOS/$(PROGRAM)\"
 endif
 else
 browser_path = \"$(TARGET_DIST)/bin/$(PROGRAM)\"
 endif
 endif
 
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -467,22 +467,22 @@ def environment(env = None):
     env["LD_LIBRARY_PATH"] = ldLibraryPath
 
   return env
 
 ###############
 # RUN THE APP #
 ###############
 
-def runApp(testURL, env, app, profileDir, extraArgs):
-  if (IS_TEST_BUILD):
+def runApp(testURL, env, app, profileDir, extraArgs, runSSLTunnel = False):
+  if IS_TEST_BUILD and runSSLTunnel:
     # create certificate database for the profile
     certificateStatus = fillCertificateDB(profileDir)
     if certificateStatus != 0:
-      log.info("ERROR FAIL Certificate integration")
+      log.info("TEST-UNEXPECTED-FAIL | Certificate integration failed")
       return certificateStatus
   
     # start ssltunnel to provide https:// URLs capability
     ssltunnel = DIST_BIN + "/ssltunnel" + BIN_SUFFIX
     ssltunnelProcess = Process(ssltunnel, [os.path.join(PROFILE_DIR, "ssltunnel.cfg")], environment())
     log.info("SSL tunnel pid: %d", ssltunnelProcess.pid)
   
   "Run the app, returning the time at which it was started."
@@ -500,23 +500,24 @@ def runApp(testURL, env, app, profileDir
     args.append("-foreground")
 
   if IS_CYGWIN:
     profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
   else:
     profileDirectory = profileDir + "/"
 
   args.extend(("-no-remote", "-profile", profileDirectory))
-  if IS_CAMINO:
-    args.extend(("-url", testURL))
-  else:
-    args.append((testURL))
+  if testURL is not None:
+    if IS_CAMINO:
+      args.extend(("-url", testURL))
+    else:
+      args.append((testURL))
   args.extend(extraArgs)
   proc = Process(cmd, args, env = environment(env))
   log.info("Application pid: %d", proc.pid)
   status = proc.wait()
   if status != 0:
-    log.info("ERROR FAIL Exited with code %d during test run", status)
+    log.info("TEST-UNEXPECTED-FAIL | Exited with code %d during test run", status)
 
-  if (IS_TEST_BUILD):
+  if IS_TEST_BUILD and runSSLTunnel:
     ssltunnelProcess.kill()
   
   return start
--- a/layout/tools/reftest/Makefile.in
+++ b/layout/tools/reftest/Makefile.in
@@ -47,18 +47,60 @@ MODULE		= reftest
 EXTRA_COMPONENTS= \
 		reftest-cmdline.js \
 		$(NULL)
 
 CHROME_DEPS	= \
 		autoconf.js \
 		$(NULL)
 
+ifdef XPI_NAME
+DIST_FILES = install.rdf
+
+# Used in install.rdf
+USE_EXTENSION_MANIFEST=1
+endif
+
 include $(topsrcdir)/config/rules.mk
 
+# We're installing to _tests/reftest
+TARGET_DEPTH = ../..
+include $(topsrcdir)/build/automation-build.mk
+
+_DEST_DIR = $(DEPTH)/_tests/reftest
+
+# We want to get an extension-packaged version of reftest as well, 
+# so this seems to be the simplest way to make that happen.
+ifndef XPI_NAME
+make-xpi:
+	+$(MAKE) -C $(CURDIR) libs XPI_NAME=reftest
+copy-harness: make-xpi
+libs:: copy-harness
+endif
+
+_HARNESS_FILES = \
+  $(srcdir)/runreftest.py \
+  automation.py \
+  $(NULL)
+
+GARBAGE += automation.py
+
+$(_DEST_DIR):
+	$(NSINSTALL) -D $@
+
+$(_HARNESS_FILES): $(_DEST_DIR)
+
+# copy harness and the reftest extension bits to $(_DEST_DIR)
+copy-harness: $(_HARNESS_FILES)
+	$(INSTALL) $(_HARNESS_FILES) $(_DEST_DIR)
+	(cd $(DIST)/xpi-stage && tar $(TAR_CREATE_FLAGS) - reftest) | (cd $(_DEST_DIR) && tar -xf -)
+	$(INSTALL) $(DIST)/bin/components/httpd.js $(_DEST_DIR)/reftest/components
+	$(INSTALL) $(DIST)/bin/components/test_necko.xpt $(_DEST_DIR)/reftest/components
+
+
 autoconf.js: $(DEPTH)/config/autoconf.mk Makefile
 	echo 'var gAutoconfVars = {' > $@
 	# strip comments, escape \ and ", use only assignment lines that
 	# don't end in backslashes and don't have variables, and then
 	# convert to JS properties
 	cat $< | egrep -v "^export " | perl -p -e 's/#.*//;s,\\,\\\\,g;s,",\\",g' | grep '=' | grep -v '\\$$' | grep -v '\$$' | perl -p -e 's/[ \t]*:?= *(.*)/: "\1",/' >> $@
 	echo 'dummy_var: null' >> $@ # to avoid trailing comma
 	echo '};' >> $@
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/install.rdf
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>reftest@mozilla.org</em:id>
+    <em:version>1.0</em:version>
+    <em:targetApplication>
+      <Description>
+        <em:id>toolkit@mozilla.org</em:id>
+#expand        <em:minVersion>__MOZILLA_VERSION_U__</em:minVersion>
+#expand        <em:maxVersion>__MOZILLA_VERSION_U__</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+    <!-- Front End MetaData -->
+    <em:name>Reftest</em:name>
+    <em:description>Run layout comparison tests.</em:description>
+    <em:creator>L. David Baron</em:creator>
+  </Description>
+</RDF>
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/runreftest.py
@@ -0,0 +1,143 @@
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Ted Mielczarek <ted.mielczarek@gmail.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""
+Runs the reftest test harness.
+"""
+
+import sys, shutil, os, os.path
+SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
+sys.path.append(SCRIPT_DIRECTORY)
+import automation
+from optparse import OptionParser
+from tempfile import mkdtemp
+
+oldcwd = os.getcwd()
+os.chdir(SCRIPT_DIRECTORY)
+
+def getFullPath(path):
+  "Get an absolute path relative to oldcwd."
+  return os.path.normpath(os.path.join(oldcwd, os.path.expanduser(path)))
+
+def createReftestProfile(profileDir):
+  "Sets up a clean profile for reftest."
+
+  # Start with a clean slate.
+  shutil.rmtree(profileDir, True)
+  os.mkdir(profileDir)
+  # reftest should only need the dump pref set
+  prefsFile = open(os.path.join(profileDir, "user.js"), "w")
+  prefsFile.write("""user_pref("browser.dom.window.dump.enabled", true);
+""")
+  prefsFile.close()
+  # install the reftest extension bits into the profile
+  profileExtensionsPath = os.path.join(profileDir, "extensions")
+  os.mkdir(profileExtensionsPath)
+  reftestExtensionPath = os.path.join(SCRIPT_DIRECTORY, "reftest")
+  extFile = open(os.path.join(profileExtensionsPath, "reftest@mozilla.org"), "w")
+  extFile.write(reftestExtensionPath)
+  extFile.close()
+
+def main():
+  parser = OptionParser()
+  parser.add_option("--appname",
+                    action = "store", type = "string", dest = "app",
+                    default = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP),
+                    help = "absolute path to application, overriding default")
+  parser.add_option("--extra-profile-file",
+                    action = "append", dest = "extraProfileFiles",
+                    default = [],
+                    help = "copy specified files/dirs to testing profile")
+  options, args = parser.parse_args()
+  
+  if len(args) != 1:
+    print >>sys.stderr, "No reftest.list specified."
+    sys.exit(1)
+
+  options.app = getFullPath(options.app)
+  if not os.path.exists(options.app):
+    print """Error: Path %(app)s doesn't exist.
+Are you executing $objdir/_tests/reftest/runreftest.py?""" \
+        % {"app": options.app}
+    sys.exit(1)
+
+  profileDir = None
+  try:
+    profileDir = mkdtemp()
+    createReftestProfile(profileDir)
+    copyExtraFilesToProfile(options, profileDir)
+
+    # browser environment
+    browserEnv = dict(os.environ)
+
+    # These variables are necessary for correct application startup; change
+    # via the commandline at your own risk.
+    browserEnv["NO_EM_RESTART"] = "1"
+    browserEnv["XPCOM_DEBUG_BREAK"] = "warn"
+    appDir = os.path.dirname(options.app)
+    if automation.UNIXISH:
+      browserEnv["LD_LIBRARY_PATH"] = appDir
+      browserEnv["MOZILLA_FIVE_HOME"] = appDir
+      browserEnv["GNOME_DISABLE_CRASH_DIALOG"] = "1"
+
+    # run once with -silent to let the extension manager do its thing
+    # and then exit the app
+    start = automation.runApp(None, browserEnv, options.app,
+                              profileDir,
+                              extraArgs = ["-silent"])
+    # then again to actually run reftest
+    reftestlist = getFullPath(args[0])
+    start = automation.runApp(None, browserEnv, options.app,
+                              profileDir,
+                              extraArgs = ["-reftest", reftestlist])
+  finally:
+    if profileDir is not None:
+      shutil.rmtree(profileDir)
+
+def copyExtraFilesToProfile(options, profileDir):
+  "Copy extra files or dirs specified on the command line to the testing profile."
+  for f in options.extraProfileFiles:
+    abspath = getFullPath(f)
+    dest = os.path.join(profileDir, os.path.basename(abspath))
+    if os.path.isdir(abspath):
+      shutil.copytree(abspath, dest)
+    else:
+      shutil.copy(abspath, dest)
+
+if __name__ == "__main__":
+  main()
--- a/testing/mochitest/runtests.py.in
+++ b/testing/mochitest/runtests.py.in
@@ -344,17 +344,17 @@ Are you executing $objdir/_tests/testing
       testURL += "?" + "&".join(urlOpts)
 
   browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE
 
   if options.fatalAssertions:
     browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 
   start = automation.runApp(testURL, browserEnv, options.app, PROFILE_DIRECTORY,
-                            options.browserArgs)
+                            options.browserArgs, runSSLTunnel = True)
 
   # Server's no longer needed, and perhaps more importantly, anything it might
   # spew to console shouldn't disrupt the leak information table we print next.
   server.stop()
 
   if not os.path.exists(LEAK_REPORT_FILE):
     log.info("WARNING refcount logging is off, so leaks can't be detected!")
   else:
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -32,21 +32,21 @@
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK *****
 
 mochitest:: mochitest-plain mochitest-chrome mochitest-a11y
 
-RUN_MOCHITEST = rm -f ./test-output.log && $(PYTHON) _tests/testing/mochitest/runtests.py --autorun --close-when-done --console-level=INFO  --log-file=./test-output.log --file-level=INFO
+RUN_MOCHITEST = rm -f ./$@.log && $(PYTHON) _tests/testing/mochitest/runtests.py --autorun --close-when-done --console-level=INFO  --log-file=./$@.log --file-level=INFO
 
 ifndef NO_FAIL_ON_TEST_ERRORS
 define CHECK_TEST_ERROR
-  @errors=`grep "TEST-UNEXPECTED-" test-output.log` ;\
+  @errors=`grep "TEST-UNEXPECTED-" $@.log` ;\
   if test "$$errors" ; then \
 	  echo "$@ failed:"; \
 	  echo "$$errors"; \
 	  exit 1; \
   else \
 	  echo "$@ passed"; \
   fi
 endef
@@ -65,24 +65,34 @@ mochitest-plain:
 mochitest-chrome:
 	$(RUN_MOCHITEST) --chrome $(MOCHITEST_PATH)
 	$(CHECK_TEST_ERROR)
 
 mochitest-a11y:
 	$(RUN_MOCHITEST) --a11y $(MOCHITEST_PATH)
 	$(CHECK_TEST_ERROR)
 
+RUN_REFTEST = rm -f ./$@.log && $(PYTHON) _tests/reftest/runreftest.py $(1) | tee ./$@.log
+
+reftest:
+	$(call RUN_REFTEST,$(topsrcdir)/layout/reftests/reftest.list)
+	$(CHECK_TEST_ERROR)
+
+crashtest:
+	$(call RUN_REFTEST,$(topsrcdir)/testing/crashtest/crashtests.list)
+	$(CHECK_TEST_ERROR)
+
 # Package up the tests and test harnesses
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 PKG_STAGE = $(DIST)/test-package-stage
 
 package-tests: stage-mochitest
 	@(cd $(PKG_STAGE) && tar $(TAR_CREATE_FLAGS) - *) | bzip2 -f > $(DIST)/$(PKG_PATH)$(TEST_PACKAGE)
 
 make-stage-dir:
 	rm -rf $(PKG_STAGE) && $(NSINSTALL) -D $(PKG_STAGE) && $(NSINSTALL) -D $(PKG_STAGE)/bin && $(NSINSTALL) -D $(PKG_STAGE)/bin/components && $(NSINSTALL) -D $(PKG_STAGE)/certs
 
 stage-mochitest: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/mochitest stage-package
 
 .PHONY: mochitest mochitest-plain mochitest-chrome mochitest-a11y \
-  package-tests make-stage-dir stage-mochitest
+  reftest crashtest package-tests make-stage-dir stage-mochitest