Bug 1087567 - Add proper manifest for cppunit tests; r=ted
authorDan Minor <dminor@mozilla.com>
Tue, 28 Oct 2014 09:38:21 -0400
changeset 245536 80a4be1149a95ef2c4afca5d7c453570670af6dd
parent 245535 e92aee06a1bfbb21c871dec08776474bc73bc6fb
child 245537 f3bdbfd4e11727dc31d312a7fdcb1d042b8dd8bc
push id698
push userjlund@mozilla.com
push dateMon, 23 Mar 2015 22:08:11 +0000
treeherdermozilla-release@b0c0ae7b02a3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1087567
milestone36.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1087567 - Add proper manifest for cppunit tests; r=ted
testing/android_cppunittest_manifest.txt
testing/b2g_cppunittest_manifest.txt
testing/cppunittest.ini
testing/mach_commands.py
testing/remotecppunittests.py
testing/runcppunittests.py
testing/testsuite-targets.mk
deleted file mode 100644
--- a/testing/android_cppunittest_manifest.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# This is just a list of tests to skip, one test per line
-TestPlainTextSerializer             # Bug 919599
-TestNativeXMLHttpRequest            # Bug 919642
-mediapipeline_unittest              # Bug 919646
-TestStartupCache                    # Bug 929655
-TestStartupCacheTelemetry.manifest  # Bug 929655
-TestStartupCacheTelemetry.js        # Bug 929655
deleted file mode 100644
--- a/testing/b2g_cppunittest_manifest.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-# This is just a list of tests to skip, one test per line
-jsapi-tests                         # Bug 1068946
-TestPLDHash                         # Bug 1038197
-TestTXMgr                           # Bug 919595
-TestPlainTextSerializer             # Bug 919599
-TestNativeXMLHttpRequest            # Bug 919642
-mediapipeline_unittest              # Bug 919646
-TestStartupCache                    # Bug 929655
-TestStartupCacheTelemetry.manifest  # Bug 929655
-TestStartupCacheTelemetry.js        # Bug 929655
-TestCSPParser                       # Bug 1054246
-TestDeadlockDetector                # Bug 1054249
-test_deadlock_detector              # Bug 1054249
-TestTArray                          # Bug 1054251
-TestAudioBuffers                    # Bug 1062937
new file mode 100644
--- /dev/null
+++ b/testing/cppunittest.ini
@@ -0,0 +1,122 @@
+[ShowAlignments]
+[ShowSSEConfig]
+[TestAppShellSteadyState]
+[TestArrayUtils]
+[TestAtomics]
+[TestAudioBuffers]
+skip-if = os == 'b2g'  # Bug 1062937
+[TestAudioChannelService]
+[TestAudioEventTimeline]
+[TestAudioMixer]
+[TestAutoPtr]
+[TestAutoRef]
+[TestBinarySearch]
+[TestBind]
+[TestBloomFilter]
+[TestCOM]
+run-if = os == 'win'
+[TestCOMArray]
+[TestCOMPtr]
+[TestCOMPtrEq]
+[TestCSPParser]
+skip-if = os == 'b2g' # Bug 1054246
+[TestCasting]
+[TestCeilingFloor]
+[TestCertDB]
+[TestCheckedInt]
+[TestCookie]
+[TestCountPopulation]
+[TestCountZeroes]
+[TestDeadlockDetector]
+skip-if = os == 'b2g'  # Bug 1054249
+[TestDeadlockDetectorScalability]
+[TestDeque]
+[TestDllInterceptor]
+run-if = os == 'win'
+[TestEndian]
+[TestEnumSet]
+[TestFile]
+[TestFloatingPoint]
+[TestGetURL]
+[TestHashtables]
+[TestID]
+[TestIntegerPrintfMacros]
+[TestJSONWriter]
+[TestJemalloc]
+[TestLineBreak]
+[TestMacroArgs]
+[TestMacroForEach]
+[TestMaybe]
+[TestNativeXMLHttpRequest]
+skip-if = os == 'b2g' || os == 'android'  #Bug 919642
+[TestNsRefPtr]
+[TestNtPathToDosPath]
+run-if = os == 'win'
+[TestObserverArray]
+[TestObserverService]
+[TestPLDHash]
+skip-if = os == 'b2g'  #Bug 1038197
+[TestPair]
+[TestPipe]
+[TestPlainTextSerializer]
+skip-if = os == 'b2g' || os == 'android'  #Bug 919599
+[TestPoisonArea]
+[TestRefPtr]
+[TestRollingMean]
+[TestSHA1]
+[TestSTSParser]
+[TestSplayTree]
+[TestStartupCache]
+skip-if = os == 'b2g' || os == 'android'  # Bug 929655
+support-files = TestStartupCacheTelemetry.js TestStartupCacheTelemetry.manifest
+[TestStringAPI]
+[TestSyncRunnable]
+[TestTArray]
+skip-if = os == 'b2g'  # Bug 1054251
+[TestTXMgr]
+skip-if = os == 'b2g'  #Bug 919595
+[TestTextFormatter]
+[TestThreadUtils]
+[TestTypeTraits]
+[TestTypedEnum]
+[TestUDPSocket]
+[TestUniquePtr]
+[TestVolatileBuffer]
+[TestWeakPtr]
+[TestWebGLElementArrayCache]
+[buffered_stun_socket_unittest]
+[ice_unittest]
+[jsapi-tests]
+skip-if = os == 'b2g'  #Bug 1068946
+[mediaconduit_unittests]
+[mediapipeline_unittest]
+skip-if = os == 'b2g' || os == 'android'  # Bug 919646
+[nrappkit_unittest]
+[rlogringbuffer_unittest]
+[runnable_utils_unittest]
+[sctp_unittest]
+[sdp_unittests]
+[signaling_unittests]
+[simpletokenbucket_unittest]
+[sockettransportservice_unittest]
+[test_AsXXX_helpers]
+[test_IHistory]
+[test_StatementCache]
+[test_asyncStatementExecution_transaction]
+[test_async_callbacks_with_spun_event_loops]
+[test_audio]
+[test_binding_params]
+[test_deadlock_detector]
+skip-if = os == 'b2g'  # Bug 1054249
+[test_file_perms]
+[test_latency]
+[test_mutex]
+[test_sanity]
+[test_service_init_background_thread]
+[test_statement_scoper]
+[test_tone]
+[test_transaction_helper]
+[test_true_async]
+[test_unlock_notify]
+[transport_unittests]
+[turn_unittest]
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -248,37 +248,38 @@ class MachCommands(MachCommandBase):
     @Command('cppunittest', category='testing',
         description='Run cpp unit tests (C++ tests).')
     @CommandArgument('test_files', nargs='*', metavar='N',
         help='Test to run. Can be specified as one or more files or ' \
             'directories, or omitted. If omitted, the entire test suite is ' \
             'executed.')
 
     def run_cppunit_test(self, **params):
+        import mozinfo
         from mozlog.structured import commandline
         import runcppunittests as cppunittests
 
         log = commandline.setup_logging("cppunittest",
                                         {},
                                         {"tbpl": sys.stdout})
 
         if len(params['test_files']) == 0:
             testdir = os.path.join(self.distdir, 'cppunittests')
-            progs = cppunittests.extract_unittests_from_args([testdir], None)
+            tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info)
         else:
-            progs = cppunittests.extract_unittests_from_args(params['test_files'], None)
+            tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info)
 
         # See if we have crash symbols
         symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')
         if not os.path.isdir(symbols_path):
             symbols_path = None
 
         tester = cppunittests.CPPUnitTests()
         try:
-            result = tester.run_tests(progs, self.bindir, symbols_path, interactive=True)
+            result = tester.run_tests(tests, self.bindir, symbols_path, interactive=True)
         except Exception as e:
             log.error("Caught exception running cpp unit tests: %s" % str(e))
             result = False
 
         return 0 if result else 1
 
 @CommandProvider
 class CheckSpiderMonkeyCommand(MachCommandBase):
--- a/testing/remotecppunittests.py
+++ b/testing/remotecppunittests.py
@@ -249,17 +249,21 @@ def main():
             sys.exit(1)
 
     log = structured.commandline.setup_logging("remotecppunittests",
                                                options,
                                                {"tbpl": sys.stdout})
 
 
     options.xre_path = os.path.abspath(options.xre_path)
-    progs = cppunittests.extract_unittests_from_args(args, options.manifest_file)
+    if options.with_b2g_emulator:
+        environ = {'os': 'b2g'}
+    else:
+        environ = {'os': 'android'}
+    progs = cppunittests.extract_unittests_from_args(args, environ)
     tester = RemoteCPPUnitTests(dm, options, progs)
     try:
         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
     except Exception, e:
         log.error(str(e))
         result = False
     if options.with_b2g_emulator:
         runner.cleanup()
--- a/testing/runcppunittests.py
+++ b/testing/runcppunittests.py
@@ -2,16 +2,17 @@
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import with_statement
 import sys, os, tempfile, shutil
 from optparse import OptionParser
+import manifestparser
 import mozprocess
 import mozinfo
 import mozcrash
 import mozfile
 from contextlib import contextmanager
 from mozlog import structured
 from subprocess import PIPE
 
@@ -169,56 +170,57 @@ class CPPUnittestOptions(OptionParser):
                         action = "store", type = "string", dest = "symbols_path",
                         default = None,
                         help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
         self.add_option("--skip-manifest",
                         action = "store", type = "string", dest = "manifest_file",
                         default = None,
                         help = "absolute path to a manifest file")
 
-def extract_unittests_from_args(args, manifest_file):
+def extract_unittests_from_args(args, environ):
     """Extract unittests from args, expanding directories as needed"""
-    progs = []
-
-    # Known files commonly packaged with the cppunittests that are not tests
-    skipped_progs = set(['.mkdir.done', 'remotecppunittests.py', 'runcppunittests.py', 'runcppunittests.pyc'])
-
-    if manifest_file:
-        skipped_progs.add(os.path.basename(manifest_file))
-        with open(manifest_file) as f:
-            for line in f:
-                # strip out comment, if any
-                prog = line.split('#')[0]
-                if prog:
-                    skipped_progs.add(prog.strip())
-
+    mp = manifestparser.TestManifest(strict=True)
+    tests = []
     for p in args:
         if os.path.isdir(p):
-            progs.extend([os.path.abspath(os.path.join(p, x)) for x in os.listdir(p) if not x in skipped_progs])
-        elif p not in skipped_progs:
-            progs.append(os.path.abspath(p))
+            try:
+                mp.read(os.path.join(p, 'cppunittest.ini'))
+            except IOError:
+                tests.extend([os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)])
+        else:
+            tests.append(os.path.abspath(p))
 
-    return progs
+    # we skip the existence check here because not all tests are built
+    # for all platforms (and it will fail on Windows anyway)
+    if mozinfo.isWin:
+        tests.extend([test['path'] + '.exe' for test in mp.active_tests(exists=False, disabled=False, **environ)])
+    else:
+        tests.extend([test['path'] for test in mp.active_tests(exists=False, disabled=False, **environ)])
+
+    # skip non-existing tests
+    tests = [test for test in tests if os.path.isfile(test)]
+
+    return tests
 
 def main():
     parser = CPPUnittestOptions()
     structured.commandline.add_logging_group(parser)
     options, args = parser.parse_args()
     if not args:
         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
         sys.exit(1)
     if not options.xre_path:
         print >>sys.stderr, """Error: --xre-path is required"""
         sys.exit(1)
 
     log = structured.commandline.setup_logging("cppunittests",
                                                options,
                                                {"tbpl": sys.stdout})
 
-    progs = extract_unittests_from_args(args, options.manifest_file)
+    progs = extract_unittests_from_args(args, mozinfo.info)
     options.xre_path = os.path.abspath(options.xre_path)
     if mozinfo.isMac:
         options.xre_path = os.path.join(os.path.dirname(options.xre_path), 'Resources')
     tester = CPPUnitTests()
 
     try:
         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
     except Exception as e:
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -500,27 +500,22 @@ endif
 
 stage-cppunittests: make-stage-dir
 	$(NSINSTALL) -D $(PKG_STAGE)/cppunittests
 ifdef STRIP_CPP_TESTS
 	$(foreach bin,$(CPP_UNIT_TEST_BINS),$(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) $(bin) $(bin:$(DIST)/%=$(PKG_STAGE)/%);)
 else
 	cp -RL $(DIST)/cppunittests $(PKG_STAGE)
 endif
-	$(NSINSTALL) $(topsrcdir)/testing/runcppunittests.py $(PKG_STAGE)/cppunittests
-	$(NSINSTALL) $(topsrcdir)/testing/remotecppunittests.py $(PKG_STAGE)/cppunittests
-ifeq ($(MOZ_WIDGET_TOOLKIT),android)
-	$(NSINSTALL) $(topsrcdir)/testing/android_cppunittest_manifest.txt $(PKG_STAGE)/cppunittests
-endif
-ifeq ($(MOZ_WIDGET_TOOLKIT),gonk)
-	$(NSINSTALL) $(topsrcdir)/testing/b2g_cppunittest_manifest.txt $(PKG_STAGE)/cppunittests
-endif
+	cp $(topsrcdir)/testing/runcppunittests.py $(PKG_STAGE)/cppunittests
+	cp $(topsrcdir)/testing/remotecppunittests.py $(PKG_STAGE)/cppunittests
+	cp $(topsrcdir)/testing/cppunittest.ini $(PKG_STAGE)/cppunittests
 ifeq ($(MOZ_DISABLE_STARTUPCACHE),)
-	$(NSINSTALL) $(topsrcdir)/startupcache/test/TestStartupCacheTelemetry.js $(PKG_STAGE)/cppunittests
-	$(NSINSTALL) $(topsrcdir)/startupcache/test/TestStartupCacheTelemetry.manifest $(PKG_STAGE)/cppunittests
+	cp $(topsrcdir)/startupcache/test/TestStartupCacheTelemetry.js $(PKG_STAGE)/cppunittests
+	cp $(topsrcdir)/startupcache/test/TestStartupCacheTelemetry.manifest $(PKG_STAGE)/cppunittests
 endif
 ifdef STRIP_CPP_TESTS
 	$(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) $(DIST)/bin/jsapi-tests$(BIN_SUFFIX) $(PKG_STAGE)/cppunittests/jsapi-tests$(BIN_SUFFIX)
 else
 	cp -RL $(DIST)/bin/jsapi-tests$(BIN_SUFFIX) $(PKG_STAGE)/cppunittests
 endif
 
 stage-jittest: make-stage-dir