Bug 1190474 - Allow test-specific timeouts to be specified in cppunittest.ini, r=chmanchester
☠☠ backed out by 34ab0b8678b2 ☠ ☠
authorJonathan Griffin <jgriffin@mozilla.com>
Wed, 12 Aug 2015 16:25:56 -0700
changeset 257521 bedff4a78d9a62c3eaaff7abec8d77956a894b40
parent 257520 c67e052b3db82f897f214f801e5caba207c67973
child 257522 718d9ac7f69705737a3cc5fb87541c421fa031a7
push id29221
push userryanvm@gmail.com
push dateThu, 13 Aug 2015 14:43:44 +0000
treeherdermozilla-central@0cddd6a6565a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1190474
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1190474 - Allow test-specific timeouts to be specified in cppunittest.ini, r=chmanchester
testing/mach_commands.py
testing/remotecppunittests.py
testing/runcppunittests.py
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -294,19 +294,20 @@ class MachCommands(MachCommandBase):
         import runcppunittests as cppunittests
 
         log = commandline.setup_logging("cppunittest",
                                         {},
                                         {"tbpl": sys.stdout})
 
         if len(params['test_files']) == 0:
             testdir = os.path.join(self.distdir, 'cppunittests')
-            tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info)
+            manifest = os.path.join(self.topsrcdir, 'testing', 'cppunittest.ini')
+            tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info, manifest)
         else:
-            tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info)
+            tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info, None)
 
         # See if we have crash symbols
         symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')
         if not os.path.isdir(symbols_path):
             symbols_path = None
 
         tester = cppunittests.CPPUnitTests()
         try:
--- a/testing/remotecppunittests.py
+++ b/testing/remotecppunittests.py
@@ -109,34 +109,37 @@ class RemoteCPPUnitTests(cppunittests.CP
                     env[envdef_parts[0]] = envdef_parts[1]
                 elif len(envdef_parts) == 1:
                     env[envdef_parts[0]] = ""
                 else:
                     self.log.warning("invalid --addEnv option skipped: %s" % envdef)
 
         return env
 
-    def run_one_test(self, prog, env, symbols_path=None, interactive=False):
+    def run_one_test(self, prog, env, symbols_path=None, interactive=False,
+                     timeout_factor=1):
         """
         Run a single C++ unit test program remotely.
 
         Arguments:
         * prog: The path to the test program to run.
         * env: The environment to use for running the program.
         * symbols_path: A path to a directory containing Breakpad-formatted
                         symbol files for producing stack traces on crash.
+        * timeout_factor: An optional test-specific timeout multiplier.
 
         Return True if the program exits with a zero status, False otherwise.
         """
         basename = os.path.basename(prog)
         remote_bin = posixpath.join(self.remote_bin_dir, basename)
         self.log.test_start(basename)
         buf = StringIO.StringIO()
+        test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
         returncode = self.device.shell([remote_bin], buf, env=env, cwd=self.remote_home_dir,
-                                       timeout=cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT)
+                                       timeout=test_timeout)
         self.log.process_output(basename, "\n%s" % buf.getvalue(),
                                 command=[remote_bin])
         with mozfile.TemporaryDirectory() as tempdir:
             self.device.getDirectory(self.remote_home_dir, tempdir)
             if mozcrash.check_for_crashes(tempdir, symbols_path,
                                           test_name=basename):
                 self.log.test_end(basename, status='CRASH', expected='PASS')
                 return False
@@ -249,18 +252,20 @@ def main():
             print "Error: you must provide a device IP to connect to via the --deviceIP option"
             sys.exit(1)
 
     log = mozlog.commandline.setup_logging("remotecppunittests", options,
                                            {"tbpl": sys.stdout})
 
     options.xre_path = os.path.abspath(options.xre_path)
     cppunittests.update_mozinfo()
-    progs = cppunittests.extract_unittests_from_args(args, mozinfo.info)
-    tester = RemoteCPPUnitTests(dm, options, progs)
+    progs = cppunittests.extract_unittests_from_args(args,
+                                                     mozinfo.info,
+                                                     options.manifest_path)
+    tester = RemoteCPPUnitTests(dm, options, [item[0] for item in progs])
     try:
         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
     except Exception, e:
         log.error(str(e))
         result = False
     if options.with_b2g_emulator:
         runner.cleanup()
         runner.wait()
--- a/testing/runcppunittests.py
+++ b/testing/runcppunittests.py
@@ -19,25 +19,27 @@ from subprocess import PIPE
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 
 class CPPUnitTests(object):
     # Time (seconds) to wait for test process to complete
     TEST_PROC_TIMEOUT = 900
     # Time (seconds) in which process will be killed if it produces no output.
     TEST_PROC_NO_OUTPUT_TIMEOUT = 300
 
-    def run_one_test(self, prog, env, symbols_path=None, interactive=False):
+    def run_one_test(self, prog, env, symbols_path=None, interactive=False,
+                     timeout_factor=1):
         """
         Run a single C++ unit test program.
 
         Arguments:
         * prog: The path to the test program to run.
         * env: The environment to use for running the program.
         * symbols_path: A path to a directory containing Breakpad-formatted
                         symbol files for producing stack traces on crash.
+        * timeout_factor: An optional test-specific timeout multiplier.
 
         Return True if the program exits with a zero status, False otherwise.
         """
         basename = os.path.basename(prog)
         self.log.test_start(basename)
         with mozfile.TemporaryDirectory() as tempdir:
             if interactive:
                 # For tests run locally, via mach, print output directly
@@ -48,17 +50,18 @@ class CPPUnitTests(object):
             else:
                 proc = mozprocess.ProcessHandler([prog],
                                                  cwd=tempdir,
                                                  env=env,
                                                  storeOutput=True,
                                                  processOutputLine=lambda _: None)
             #TODO: After bug 811320 is fixed, don't let .run() kill the process,
             # instead use a timeout in .wait() and then kill to get a stack.
-            proc.run(timeout=CPPUnitTests.TEST_PROC_TIMEOUT,
+            test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
+            proc.run(timeout=test_timeout,
                      outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT)
             proc.wait()
             if proc.output:
                 output = "\n%s" % "\n".join(proc.output)
                 self.log.process_output(proc.pid, output, command=[prog])
             if proc.timedOut:
                 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
                 self.log.test_end(basename, status='TIMEOUT', expected='PASS',
@@ -128,32 +131,35 @@ class CPPUnitTests(object):
 
         return env
 
     def run_tests(self, programs, xre_path, symbols_path=None, interactive=False):
         """
         Run a set of C++ unit test programs.
 
         Arguments:
-        * programs: An iterable containing paths to test programs.
+        * programs: An iterable containing (test path, test timeout factor) tuples
         * xre_path: A path to a directory containing a XUL Runtime Environment.
         * symbols_path: A path to a directory containing Breakpad-formatted
                         symbol files for producing stack traces on crash.
 
         Returns True if all test programs exited with a zero status, False
         otherwise.
         """
         self.xre_path = xre_path
         self.log = mozlog.get_default_logger()
         self.log.suite_start(programs)
         env = self.build_environment()
         pass_count = 0
         fail_count = 0
         for prog in programs:
-            single_result = self.run_one_test(prog, env, symbols_path, interactive)
+            test_path = prog[0]
+            timeout_factor = prog[1]
+            single_result = self.run_one_test(test_path, env, symbols_path,
+                                              interactive, timeout_factor)
             if single_result:
                 pass_count += 1
             else:
                 fail_count += 1
         self.log.suite_end()
 
         # Mozharness-parseable summary formatting.
         self.log.info("Result summary:")
@@ -167,43 +173,51 @@ class CPPUnittestOptions(OptionParser):
         self.add_option("--xre-path",
                         action = "store", type = "string", dest = "xre_path",
                         default = None,
                         help = "absolute path to directory containing XRE (probably xulrunner)")
         self.add_option("--symbols-path",
                         action = "store", type = "string", dest = "symbols_path",
                         default = None,
                         help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
-        self.add_option("--skip-manifest",
-                        action = "store", type = "string", dest = "manifest_file",
+        self.add_option("--manifest-path",
+                        action = "store", type = "string", dest = "manifest_path",
                         default = None,
-                        help = "absolute path to a manifest file")
+                        help = "path to test manifest, if different from the path to test binaries")
 
-def extract_unittests_from_args(args, environ):
+def extract_unittests_from_args(args, environ, manifest_path):
     """Extract unittests from args, expanding directories as needed"""
     mp = manifestparser.TestManifest(strict=True)
     tests = []
-    for p in args:
-        if os.path.isdir(p):
-            try:
-                mp.read(os.path.join(p, 'cppunittest.ini'))
-            except IOError:
-                tests.extend([os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)])
-        else:
-            tests.append(os.path.abspath(p))
+    binary_path = None
+
+    if manifest_path:
+        mp.read(manifest_path)
+        binary_path = os.path.abspath(args[0])
+    else:
+        for p in args:
+            if os.path.isdir(p):
+                try:
+                    mp.read(os.path.join(p, 'cppunittest.ini'))
+                except IOError:
+                    tests.extend([(os.path.abspath(os.path.join(p, x)), 1) for x in os.listdir(p)])
+            else:
+                tests.append((os.path.abspath(p), 1))
 
     # we skip the existence check here because not all tests are built
     # for all platforms (and it will fail on Windows anyway)
-    if mozinfo.isWin:
-        tests.extend([test['path'] + '.exe' for test in mp.active_tests(exists=False, disabled=False, **environ)])
+    active_tests = mp.active_tests(exists=False, disabled=False, **environ)
+    suffix = '.exe' if mozinfo.isWin else ''
+    if binary_path:
+        tests.extend([(os.path.join(binary_path, test['relpath'] + suffix), int(test.get('requesttimeoutfactor', 1))) for test in active_tests])
     else:
-        tests.extend([test['path'] for test in mp.active_tests(exists=False, disabled=False, **environ)])
+        tests.extend([(test['path'] + suffix, int(test.get('requesttimeoutfactor', 1))) for test in active_tests])
 
     # skip non-existing tests
-    tests = [test for test in tests if os.path.isfile(test)]
+    tests = [test for test in tests if os.path.isfile(test[0])]
 
     return tests
 
 def update_mozinfo():
     """walk up directories to find mozinfo.json update the info"""
     path = SCRIPT_DIR
     dirs = set()
     while path != os.path.expanduser('~'):
@@ -218,22 +232,25 @@ def main():
     mozlog.commandline.add_logging_group(parser)
     options, args = parser.parse_args()
     if not args:
         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
         sys.exit(1)
     if not options.xre_path:
         print >>sys.stderr, """Error: --xre-path is required"""
         sys.exit(1)
+    if options.manifest_path and len(args) > 1:
+        print >>sys.stderr, "Error: multiple arguments not supported with --test-manifest"
+        sys.exit(1)
 
     log = mozlog.commandline.setup_logging("cppunittests", options,
                                            {"tbpl": sys.stdout})
 
     update_mozinfo()
-    progs = extract_unittests_from_args(args, mozinfo.info)
+    progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
     options.xre_path = os.path.abspath(options.xre_path)
     if mozinfo.isMac:
         options.xre_path = os.path.join(os.path.dirname(options.xre_path), 'Resources')
     tester = CPPUnitTests()
 
     try:
         result = tester.run_tests(progs, options.xre_path, options.symbols_path)
     except Exception as e: