Bug 1087567 - Add proper manifest for cppunit tests; r=ted
authorDan Minor <dminor@mozilla.com>
Tue, 28 Oct 2014 09:38:21 -0400
changeset 234753 80a4be1149a95ef2c4afca5d7c453570670af6dd
parent 234752 e92aee06a1bfbb21c871dec08776474bc73bc6fb
child 234754 f3bdbfd4e11727dc31d312a7fdcb1d042b8dd8bc
push id7472
push userraliiev@mozilla.com
push dateMon, 12 Jan 2015 20:36:27 +0000
treeherdermozilla-aurora@300ca104f8fb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1087567
milestone36.0a1
Bug 1087567 - Add proper manifest for cppunit tests; r=ted
testing/android_cppunittest_manifest.txt
testing/b2g_cppunittest_manifest.txt
testing/cppunittest.ini
testing/mach_commands.py
testing/remotecppunittests.py
testing/runcppunittests.py
testing/testsuite-targets.mk
deleted file mode 100644
--- a/testing/android_cppunittest_manifest.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# This is just a list of tests to skip, one test per line
-TestPlainTextSerializer             # Bug 919599
-TestNativeXMLHttpRequest            # Bug 919642
-mediapipeline_unittest              # Bug 919646
-TestStartupCache                    # Bug 929655
-TestStartupCacheTelemetry.manifest  # Bug 929655
-TestStartupCacheTelemetry.js        # Bug 929655
deleted file mode 100644
--- a/testing/b2g_cppunittest_manifest.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-# This is just a list of tests to skip, one test per line
-jsapi-tests                         # Bug 1068946
-TestPLDHash                         # Bug 1038197
-TestTXMgr                           # Bug 919595
-TestPlainTextSerializer             # Bug 919599
-TestNativeXMLHttpRequest            # Bug 919642
-mediapipeline_unittest              # Bug 919646
-TestStartupCache                    # Bug 929655
-TestStartupCacheTelemetry.manifest  # Bug 929655
-TestStartupCacheTelemetry.js        # Bug 929655
-TestCSPParser                       # Bug 1054246
-TestDeadlockDetector                # Bug 1054249
-test_deadlock_detector              # Bug 1054249
-TestTArray                          # Bug 1054251
-TestAudioBuffers                    # Bug 1062937
new file mode 100644
--- /dev/null
+++ b/testing/cppunittest.ini
@@ -0,0 +1,122 @@
+[ShowAlignments]
+[ShowSSEConfig]
+[TestAppShellSteadyState]
+[TestArrayUtils]
+[TestAtomics]
+[TestAudioBuffers]
+skip-if = os == 'b2g'  # Bug 1062937
+[TestAudioChannelService]
+[TestAudioEventTimeline]
+[TestAudioMixer]
+[TestAutoPtr]
+[TestAutoRef]
+[TestBinarySearch]
+[TestBind]
+[TestBloomFilter]
+[TestCOM]
+run-if = os == 'win'
+[TestCOMArray]
+[TestCOMPtr]
+[TestCOMPtrEq]
+[TestCSPParser]
+skip-if = os == 'b2g' # Bug 1054246
+[TestCasting]
+[TestCeilingFloor]
+[TestCertDB]
+[TestCheckedInt]
+[TestCookie]
+[TestCountPopulation]
+[TestCountZeroes]
+[TestDeadlockDetector]
+skip-if = os == 'b2g'  # Bug 1054249
+[TestDeadlockDetectorScalability]
+[TestDeque]
+[TestDllInterceptor]
+run-if = os == 'win'
+[TestEndian]
+[TestEnumSet]
+[TestFile]
+[TestFloatingPoint]
+[TestGetURL]
+[TestHashtables]
+[TestID]
+[TestIntegerPrintfMacros]
+[TestJSONWriter]
+[TestJemalloc]
+[TestLineBreak]
+[TestMacroArgs]
+[TestMacroForEach]
+[TestMaybe]
+[TestNativeXMLHttpRequest]
+skip-if = os == 'b2g' || os == 'android'  #Bug 919642
+[TestNsRefPtr]
+[TestNtPathToDosPath]
+run-if = os == 'win'
+[TestObserverArray]
+[TestObserverService]
+[TestPLDHash]
+skip-if = os == 'b2g'  #Bug 1038197
+[TestPair]
+[TestPipe]
+[TestPlainTextSerializer]
+skip-if = os == 'b2g' || os == 'android'  #Bug 919599
+[TestPoisonArea]
+[TestRefPtr]
+[TestRollingMean]
+[TestSHA1]
+[TestSTSParser]
+[TestSplayTree]
+[TestStartupCache]
+skip-if = os == 'b2g' || os == 'android'  # Bug 929655
+support-files = TestStartupCacheTelemetry.js TestStartupCacheTelemetry.manifest
+[TestStringAPI]
+[TestSyncRunnable]
+[TestTArray]
+skip-if = os == 'b2g'  # Bug 1054251
+[TestTXMgr]
+skip-if = os == 'b2g'  #Bug 919595
+[TestTextFormatter]
+[TestThreadUtils]
+[TestTypeTraits]
+[TestTypedEnum]
+[TestUDPSocket]
+[TestUniquePtr]
+[TestVolatileBuffer]
+[TestWeakPtr]
+[TestWebGLElementArrayCache]
+[buffered_stun_socket_unittest]
+[ice_unittest]
+[jsapi-tests]
+skip-if = os == 'b2g'  #Bug 1068946
+[mediaconduit_unittests]
+[mediapipeline_unittest]
+skip-if = os == 'b2g' || os == 'android'  # Bug 919646
+[nrappkit_unittest]
+[rlogringbuffer_unittest]
+[runnable_utils_unittest]
+[sctp_unittest]
+[sdp_unittests]
+[signaling_unittests]
+[simpletokenbucket_unittest]
+[sockettransportservice_unittest]
+[test_AsXXX_helpers]
+[test_IHistory]
+[test_StatementCache]
+[test_asyncStatementExecution_transaction]
+[test_async_callbacks_with_spun_event_loops]
+[test_audio]
+[test_binding_params]
+[test_deadlock_detector]
+skip-if = os == 'b2g'  # Bug 1054249
+[test_file_perms]
+[test_latency]
+[test_mutex]
+[test_sanity]
+[test_service_init_background_thread]
+[test_statement_scoper]
+[test_tone]
+[test_transaction_helper]
+[test_true_async]
+[test_unlock_notify]
+[transport_unittests]
+[turn_unittest]
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -248,37 +248,38 @@ class MachCommands(MachCommandBase):
     @Command('cppunittest', category='testing',
         description='Run cpp unit tests (C++ tests).')
     @CommandArgument('test_files', nargs='*', metavar='N',
         help='Test to run. Can be specified as one or more files or ' \
             'directories, or omitted. If omitted, the entire test suite is ' \
             'executed.')
 
     def run_cppunit_test(self, **params):
+        import mozinfo
         from mozlog.structured import commandline
         import runcppunittests as cppunittests
 
         log = commandline.setup_logging("cppunittest",
                                         {},
                                         {"tbpl": sys.stdout})
 
         if len(params['test_files']) == 0:
             testdir = os.path.join(self.distdir, 'cppunittests')
-            progs = cppunittests.extract_unittests_from_args([testdir], None)
+            tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info)
         else:
-            progs = cppunittests.extract_unittests_from_args(params['test_files'], None)
+            tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info)
 
         # See if we have crash symbols
         symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')
         if not os.path.isdir(symbols_path):
             symbols_path = None
 
         tester = cppunittests.CPPUnitTests()
         try:
-            result = tester.run_tests(progs, self.bindir, symbols_path, interactive=True)
+            result = tester.run_tests(tests, self.bindir, symbols_path, interactive=True)
         except Exception as e:
             log.error("Caught exception running cpp unit tests: %s" % str(e))
             result = False
 
         return 0 if result else 1
 
 @CommandProvider
 class CheckSpiderMonkeyCommand(MachCommandBase):
--- a/testing/remotecppunittests.py
+++ b/testing/remotecppunittests.py
@@ -249,17 +249,21 @@ def main():
             sys.exit(1)
 
     log = structured.commandline.setup_logging("remotecppunittests",
                                                options,
                                                {"tbpl": sys.stdout})
 
 
     options.xre_path = os.path.abspath(options.xre_path)
-    progs = cppunittests.extract_unittests_from_args(args, options.manifest_file)
+    if options.with_b2g_emulator:
+        environ = {'os': 'b2g'}
+    else:
+        environ = {'os': 'android'}
+    progs = cppunittests.extract_unittests_from_args(args, environ)
     tester = RemoteCPPUnitTests(dm, options, progs)
     try:
         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
     except Exception, e:
         log.error(str(e))
         result = False
     if options.with_b2g_emulator:
         runner.cleanup()
--- a/testing/runcppunittests.py
+++ b/testing/runcppunittests.py
@@ -2,16 +2,17 @@
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import with_statement
 import sys, os, tempfile, shutil
 from optparse import OptionParser
+import manifestparser
 import mozprocess
 import mozinfo
 import mozcrash
 import mozfile
 from contextlib import contextmanager
 from mozlog import structured
 from subprocess import PIPE
 
@@ -169,56 +170,57 @@ class CPPUnittestOptions(OptionParser):
                         action = "store", type = "string", dest = "symbols_path",
                         default = None,
                         help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
         self.add_option("--skip-manifest",
                         action = "store", type = "string", dest = "manifest_file",
                         default = None,
                         help = "absolute path to a manifest file")
 
-def extract_unittests_from_args(args, manifest_file):
+def extract_unittests_from_args(args, environ):
     """Extract unittests from args, expanding directories as needed"""
-    progs = []
-
-    # Known files commonly packaged with the cppunittests that are not tests
-    skipped_progs = set(['.mkdir.done', 'remotecppunittests.py', 'runcppunittests.py', 'runcppunittests.pyc'])
-
-    if manifest_file:
-        skipped_progs.add(os.path.basename(manifest_file))
-        with open(manifest_file) as f:
-            for line in f:
-                # strip out comment, if any
-                prog = line.split('#')[0]
-                if prog:
-                    skipped_progs.add(prog.strip())
-
+    mp = manifestparser.TestManifest(strict=True)
+    tests = []
     for p in args:
         if os.path.isdir(p):
-            progs.extend([os.path.abspath(os.path.join(p, x)) for x in os.listdir(p) if not x in skipped_progs])
-        elif p not in skipped_progs:
-            progs.append(os.path.abspath(p))
+            try:
+                mp.read(os.path.join(p, 'cppunittest.ini'))
+            except IOError:
+                tests.extend([os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)])
+        else:
+            tests.append(os.path.abspath(p))
 
-    return progs
+    # we skip the existence check here because not all tests are built
+    # for all platforms (and it will fail on Windows anyway)
+    if mozinfo.isWin:
+        tests.extend([test['path'] + '.exe' for test in mp.active_tests(exists=False, disabled=False, **environ)])
+    else:
+        tests.extend([test['path'] for test in mp.active_tests(exists=False, disabled=False, **environ)])
+
+    # skip non-existing tests
+    tests = [test for test in tests if os.path.isfile(test)]
+
+    return tests
 
 def main():
     parser = CPPUnittestOptions()
     structured.commandline.add_logging_group(parser)
     options, args = parser.parse_args()
     if not args:
         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
         sys.exit(1)
     if not options.xre_path:
         print >>sys.stderr, """Error: --xre-path is required"""
         sys.exit(1)
 
     log = structured.commandline.setup_logging("cppunittests",
                                                options,
                                                {"tbpl": sys.stdout})
 
-    progs = extract_unittests_from_args(args, options.manifest_file)
+    progs = extract_unittests_from_args(args, mozinfo.info)
     options.xre_path = os.path.abspath(options.xre_path)
     if mozinfo.isMac:
         options.xre_path = os.path.join(os.path.dirname(options.xre_path), 'Resources')
     tester = CPPUnitTests()
 
     try:
         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
     except Exception as e:
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -500,27 +500,22 @@ endif
 
 stage-cppunittests: make-stage-dir
 	$(NSINSTALL) -D $(PKG_STAGE)/cppunittests
 ifdef STRIP_CPP_TESTS
 	$(foreach bin,$(CPP_UNIT_TEST_BINS),$(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) $(bin) $(bin:$(DIST)/%=$(PKG_STAGE)/%);)
 else
 	cp -RL $(DIST)/cppunittests $(PKG_STAGE)
 endif
-	$(NSINSTALL) $(topsrcdir)/testing/runcppunittests.py $(PKG_STAGE)/cppunittests
-	$(NSINSTALL) $(topsrcdir)/testing/remotecppunittests.py $(PKG_STAGE)/cppunittests
-ifeq ($(MOZ_WIDGET_TOOLKIT),android)
-	$(NSINSTALL) $(topsrcdir)/testing/android_cppunittest_manifest.txt $(PKG_STAGE)/cppunittests
-endif
-ifeq ($(MOZ_WIDGET_TOOLKIT),gonk)
-	$(NSINSTALL) $(topsrcdir)/testing/b2g_cppunittest_manifest.txt $(PKG_STAGE)/cppunittests
-endif
+	cp $(topsrcdir)/testing/runcppunittests.py $(PKG_STAGE)/cppunittests
+	cp $(topsrcdir)/testing/remotecppunittests.py $(PKG_STAGE)/cppunittests
+	cp $(topsrcdir)/testing/cppunittest.ini $(PKG_STAGE)/cppunittests
 ifeq ($(MOZ_DISABLE_STARTUPCACHE),)
-	$(NSINSTALL) $(topsrcdir)/startupcache/test/TestStartupCacheTelemetry.js $(PKG_STAGE)/cppunittests
-	$(NSINSTALL) $(topsrcdir)/startupcache/test/TestStartupCacheTelemetry.manifest $(PKG_STAGE)/cppunittests
+	cp $(topsrcdir)/startupcache/test/TestStartupCacheTelemetry.js $(PKG_STAGE)/cppunittests
+	cp $(topsrcdir)/startupcache/test/TestStartupCacheTelemetry.manifest $(PKG_STAGE)/cppunittests
 endif
 ifdef STRIP_CPP_TESTS
 	$(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) $(DIST)/bin/jsapi-tests$(BIN_SUFFIX) $(PKG_STAGE)/cppunittests/jsapi-tests$(BIN_SUFFIX)
 else
 	cp -RL $(DIST)/bin/jsapi-tests$(BIN_SUFFIX) $(PKG_STAGE)/cppunittests
 endif
 
 stage-jittest: make-stage-dir