Bug 1190474 - Allow test-specific timeouts to be specified in cppunittest.ini, r=chmanchester
authorJonathan Griffin <jgriffin@mozilla.com>
Wed, 12 Aug 2015 16:25:56 -0700
changeset 257748 8fa8ea614226b42f447e38576dfe1eef7f89f685
parent 257747 cef6a397f50b4c92d2f021c0549f674c80a49cde
child 257749 be0aec3fbdae6a641159a1bfbca333d00d58ed97
push id29226
push userryanvm@gmail.com
push dateFri, 14 Aug 2015 13:01:14 +0000
treeherdermozilla-central@1b2402247429 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1190474
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1190474 - Allow test-specific timeouts to be specified in cppunittest.ini, r=chmanchester
testing/config/mozharness/b2g_emulator_config.py
testing/mach_commands.py
testing/remotecppunittests.py
testing/runcppunittests.py
--- a/testing/config/mozharness/b2g_emulator_config.py
+++ b/testing/config/mozharness/b2g_emulator_config.py
@@ -7,17 +7,16 @@ config = {
         "cppunittest": {
             "options": [
                 "--dm_trans=adb",
                 "--symbols-path=%(symbols_path)s",
                 "--xre-path=%(xre_path)s",
                 "--addEnv",
                 "LD_LIBRARY_PATH=/vendor/lib:/system/lib:/system/b2g",
                 "--with-b2g-emulator=%(b2gpath)s",
-                "--skip-manifest=b2g_cppunittest_manifest.txt",
                 "."
             ],
             "run_filename": "remotecppunittests.py",
             "testsdir": "cppunittest"
         },
         "crashtest": {
             "options": [
                 "--adbpath=%(adbpath)s",
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -294,19 +294,20 @@ class MachCommands(MachCommandBase):
         import runcppunittests as cppunittests
 
         log = commandline.setup_logging("cppunittest",
                                         {},
                                         {"tbpl": sys.stdout})
 
         if len(params['test_files']) == 0:
             testdir = os.path.join(self.distdir, 'cppunittests')
-            tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info)
+            manifest = os.path.join(self.topsrcdir, 'testing', 'cppunittest.ini')
+            tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info, manifest)
         else:
-            tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info)
+            tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info, None)
 
         # See if we have crash symbols
         symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')
         if not os.path.isdir(symbols_path):
             symbols_path = None
 
         tester = cppunittests.CPPUnitTests()
         try:
--- a/testing/remotecppunittests.py
+++ b/testing/remotecppunittests.py
@@ -109,34 +109,37 @@ class RemoteCPPUnitTests(cppunittests.CP
                     env[envdef_parts[0]] = envdef_parts[1]
                 elif len(envdef_parts) == 1:
                     env[envdef_parts[0]] = ""
                 else:
                     self.log.warning("invalid --addEnv option skipped: %s" % envdef)
 
         return env
 
-    def run_one_test(self, prog, env, symbols_path=None, interactive=False):
+    def run_one_test(self, prog, env, symbols_path=None, interactive=False,
+                     timeout_factor=1):
         """
         Run a single C++ unit test program remotely.
 
         Arguments:
         * prog: The path to the test program to run.
         * env: The environment to use for running the program.
         * symbols_path: A path to a directory containing Breakpad-formatted
                         symbol files for producing stack traces on crash.
+        * timeout_factor: An optional test-specific timeout multiplier.
 
         Return True if the program exits with a zero status, False otherwise.
         """
         basename = os.path.basename(prog)
         remote_bin = posixpath.join(self.remote_bin_dir, basename)
         self.log.test_start(basename)
         buf = StringIO.StringIO()
+        test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
         returncode = self.device.shell([remote_bin], buf, env=env, cwd=self.remote_home_dir,
-                                       timeout=cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT)
+                                       timeout=test_timeout)
         self.log.process_output(basename, "\n%s" % buf.getvalue(),
                                 command=[remote_bin])
         with mozfile.TemporaryDirectory() as tempdir:
             self.device.getDirectory(self.remote_home_dir, tempdir)
             if mozcrash.check_for_crashes(tempdir, symbols_path,
                                           test_name=basename):
                 self.log.test_end(basename, status='CRASH', expected='PASS')
                 return False
@@ -249,18 +252,20 @@ def main():
             print "Error: you must provide a device IP to connect to via the --deviceIP option"
             sys.exit(1)
 
     log = mozlog.commandline.setup_logging("remotecppunittests", options,
                                            {"tbpl": sys.stdout})
 
     options.xre_path = os.path.abspath(options.xre_path)
     cppunittests.update_mozinfo()
-    progs = cppunittests.extract_unittests_from_args(args, mozinfo.info)
-    tester = RemoteCPPUnitTests(dm, options, progs)
+    progs = cppunittests.extract_unittests_from_args(args,
+                                                     mozinfo.info,
+                                                     options.manifest_path)
+    tester = RemoteCPPUnitTests(dm, options, [item[0] for item in progs])
     try:
         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
     except Exception, e:
         log.error(str(e))
         result = False
     if options.with_b2g_emulator:
         runner.cleanup()
         runner.wait()
--- a/testing/runcppunittests.py
+++ b/testing/runcppunittests.py
@@ -19,25 +19,27 @@ from subprocess import PIPE
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 
 class CPPUnitTests(object):
     # Time (seconds) to wait for test process to complete
     TEST_PROC_TIMEOUT = 900
     # Time (seconds) in which process will be killed if it produces no output.
     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
 
-    def run_one_test(self, prog, env, symbols_path=None, interactive=False):
+    def run_one_test(self, prog, env, symbols_path=None, interactive=False,
+                     timeout_factor=1):
         """
         Run a single C++ unit test program.
 
         Arguments:
         * prog: The path to the test program to run.
         * env: The environment to use for running the program.
         * symbols_path: A path to a directory containing Breakpad-formatted
                         symbol files for producing stack traces on crash.
+        * timeout_factor: An optional test-specific timeout multiplier.
 
         Return True if the program exits with a zero status, False otherwise.
         """
         basename = os.path.basename(prog)
         self.log.test_start(basename)
         with mozfile.TemporaryDirectory() as tempdir:
             if interactive:
                 # For tests run locally, via mach, print output directly
@@ -48,17 +50,18 @@ class CPPUnitTests(object):
             else:
                 proc = mozprocess.ProcessHandler([prog],
                                                  cwd=tempdir,
                                                  env=env,
                                                  storeOutput=True,
                                                  processOutputLine=lambda _: None)
             #TODO: After bug 811320 is fixed, don't let .run() kill the process,
             # instead use a timeout in .wait() and then kill to get a stack.
-            proc.run(timeout=CPPUnitTests.TEST_PROC_TIMEOUT,
+            test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
+            proc.run(timeout=test_timeout,
                      outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
             proc.wait()
             if proc.output:
                 output = "\n%s" % "\n".join(proc.output)
                 self.log.process_output(proc.pid, output, command=[prog])
             if proc.timedOut:
                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
                 self.log.test_end(basename, status='TIMEOUT', expected='PASS',
@@ -128,32 +131,35 @@ class CPPUnitTests(object):
 
         return env
 
     def run_tests(self, programs, xre_path, symbols_path=None, interactive=False):
         """
         Run a set of C++ unit test programs.
 
         Arguments:
-        * programs: An iterable containing paths to test programs.
+        * programs: An iterable containing (test path, test timeout factor) tuples
         * xre_path: A path to a directory containing a XUL Runtime Environment.
         * symbols_path: A path to a directory containing Breakpad-formatted
                         symbol files for producing stack traces on crash.
 
         Returns True if all test programs exited with a zero status, False
         otherwise.
         """
         self.xre_path = xre_path
         self.log = mozlog.get_default_logger()
         self.log.suite_start(programs)
         env = self.build_environment()
         pass_count = 0
         fail_count = 0
         for prog in programs:
-            single_result = self.run_one_test(prog, env, symbols_path, interactive)
+            test_path = prog[0]
+            timeout_factor = prog[1]
+            single_result = self.run_one_test(test_path, env, symbols_path,
+                                              interactive, timeout_factor)
             if single_result:
                 pass_count += 1
             else:
                 fail_count += 1
         self.log.suite_end()
 
         # Mozharness-parseable summary formatting.
         self.log.info("Result summary:")
@@ -167,43 +173,51 @@ class CPPUnittestOptions(OptionParser):
         self.add_option("--xre-path",
                         action = "store", type = "string", dest = "xre_path",
                         default = None,
                         help = "absolute path to directory containing XRE (probably xulrunner)")
         self.add_option("--symbols-path",
                         action = "store", type = "string", dest = "symbols_path",
                         default = None,
                         help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
-        self.add_option("--skip-manifest",
-                        action = "store", type = "string", dest = "manifest_file",
+        self.add_option("--manifest-path",
+                        action = "store", type = "string", dest = "manifest_path",
                         default = None,
-                        help = "absolute path to a manifest file")
+                        help = "path to test manifest, if different from the path to test binaries")
 
-def extract_unittests_from_args(args, environ):
+def extract_unittests_from_args(args, environ, manifest_path):
     """Extract unittests from args, expanding directories as needed"""
     mp = manifestparser.TestManifest(strict=True)
     tests = []
-    for p in args:
-        if os.path.isdir(p):
-            try:
-                mp.read(os.path.join(p, 'cppunittest.ini'))
-            except IOError:
-                tests.extend([os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)])
-        else:
-            tests.append(os.path.abspath(p))
+    binary_path = None
+
+    if manifest_path:
+        mp.read(manifest_path)
+        binary_path = os.path.abspath(args[0])
+    else:
+        for p in args:
+            if os.path.isdir(p):
+                try:
+                    mp.read(os.path.join(p, 'cppunittest.ini'))
+                except IOError:
+                    tests.extend([(os.path.abspath(os.path.join(p, x)), 1) for x in os.listdir(p)])
+            else:
+                tests.append((os.path.abspath(p), 1))
 
     # we skip the existence check here because not all tests are built
     # for all platforms (and it will fail on Windows anyway)
-    if mozinfo.isWin:
-        tests.extend([test['path'] + '.exe' for test in mp.active_tests(exists=False, disabled=False, **environ)])
+    active_tests = mp.active_tests(exists=False, disabled=False, **environ)
+    suffix = '.exe' if mozinfo.isWin else ''
+    if binary_path:
+        tests.extend([(os.path.join(binary_path, test['relpath'] + suffix), int(test.get('requesttimeoutfactor', 1))) for test in active_tests])
     else:
-        tests.extend([test['path'] for test in mp.active_tests(exists=False, disabled=False, **environ)])
+        tests.extend([(test['path'] + suffix, int(test.get('requesttimeoutfactor', 1))) for test in active_tests])
 
     # skip non-existing tests
-    tests = [test for test in tests if os.path.isfile(test)]
+    tests = [test for test in tests if os.path.isfile(test[0])]
 
     return tests
 
 def update_mozinfo():
     """walk up directories to find mozinfo.json update the info"""
     path = SCRIPT_DIR
     dirs = set()
     while path != os.path.expanduser('~'):
@@ -218,22 +232,25 @@ def main():
     mozlog.commandline.add_logging_group(parser)
     options, args = parser.parse_args()
     if not args:
         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
         sys.exit(1)
     if not options.xre_path:
         print >>sys.stderr, """Error: --xre-path is required"""
         sys.exit(1)
+    if options.manifest_path and len(args) > 1:
+        print >>sys.stderr, "Error: multiple arguments not supported with --test-manifest"
+        sys.exit(1)
 
     log = mozlog.commandline.setup_logging("cppunittests", options,
                                            {"tbpl": sys.stdout})
 
     update_mozinfo()
-    progs = extract_unittests_from_args(args, mozinfo.info)
+    progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
     options.xre_path = os.path.abspath(options.xre_path)
     if mozinfo.isMac:
         options.xre_path = os.path.join(os.path.dirname(options.xre_path), 'Resources')
     tester = CPPUnitTests()
 
     try:
         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
     except Exception as e: