Bug 1176004 - Make jstest's unix task runner a generator; r=sfink
authorTerrence Cole <terrence@mozilla.com>
Wed, 17 Jun 2015 13:57:02 -0700
changeset 280669 43aa1645ee307aa63c414487e708e3a0d3e68e5f
parent 280668 cf426328aee818b5a1756123243631dd55193f34
child 280670 88f5bdaf1d8e96022f5bee0f4dbd368f5b42305a
push id4932
push userjlund@mozilla.com
push dateMon, 10 Aug 2015 18:23:06 +0000
treeherdermozilla-beta@6dd5a4f5f745 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfink
bugs1176004
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1176004 - Make jstest's unix task runner a generator; r=sfink
js/src/tests/lib/tasks_unix.py
--- a/js/src/tests/lib/tasks_unix.py
+++ b/js/src/tests/lib/tasks_unix.py
@@ -1,14 +1,15 @@
 # A unix-oriented process dispatcher.  Uses a single thread with select and
 # waitpid to dispatch tasks.  This avoids several deadlocks that are possible
 # with fork/exec + threads + Python.
 
 import errno, os, select
 from datetime import datetime, timedelta
+from progressbar import ProgressBar
 from results import TestOutput
 
 class Task(object):
     def __init__(self, test, prefix, pid, stdout, stderr):
         self.test = test
         self.cmd = test.get_command(prefix)
         self.pid = pid
         self.stdout = stdout
@@ -43,23 +44,23 @@ def spawn_test(test, prefix, passthrough
 
 def total_seconds(td):
     """
     Return the total number of seconds contained in the duration as a float
     """
     return (float(td.microseconds) \
             + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
 
-def get_max_wait(tasks, results, timeout):
+def get_max_wait(tasks, timeout):
     """
     Return the maximum time we can wait before any task should time out.
     """
 
     # If we have a progress-meter, we need to wake up to update it frequently.
-    wait = results.pb.update_granularity()
+    wait = ProgressBar.update_granularity()
 
     # If a timeout is supplied, we need to wake up for the first task to
     # timeout if that is sooner.
     if timeout:
         now = datetime.now()
         timeout_delta = timedelta(seconds=timeout)
         for task in tasks:
             remaining = task.start + timeout_delta - now
@@ -129,23 +130,24 @@ def timed_out(task, timeout):
     |timeout| may be falsy, indicating an infinite timeout (in which case
     timed_out always returns False).
     """
     if timeout:
         now = datetime.now()
         return (now - task.start) > timedelta(seconds=timeout)
     return False
 
-def reap_zombies(tasks, results, timeout):
+def reap_zombies(tasks, timeout):
     """
     Search for children of this process that have finished.  If they are tasks,
     then this routine will clean up the child and send a TestOutput to the
     results channel.  This method returns a new task list that has had the ended
     tasks removed.
     """
+    finished = []
     while True:
         try:
             pid, status = os.waitpid(0, os.WNOHANG)
             if pid == 0:
                 break
         except OSError as e:
             if e.errno == errno.ECHILD:
                 break
@@ -156,48 +158,57 @@ def reap_zombies(tasks, results, timeout
         flush_input(ended.stderr, ended.err)
         os.close(ended.stdout)
         os.close(ended.stderr)
 
         returncode = os.WEXITSTATUS(status)
         if os.WIFSIGNALED(status):
             returncode = -os.WTERMSIG(status)
 
-        out = TestOutput(
-            ended.test,
-            ended.cmd,
-            ''.join(ended.out),
-            ''.join(ended.err),
-            returncode,
-            total_seconds(datetime.now() - ended.start),
-            timed_out(ended, timeout))
-        results.push(out)
-    return tasks
+        finished.append(
+            TestOutput(
+                ended.test,
+                ended.cmd,
+                ''.join(ended.out),
+                ''.join(ended.err),
+                returncode,
+                total_seconds(datetime.now() - ended.start),
+                timed_out(ended, timeout)))
+    return tasks, finished
 
-def kill_undead(tasks, results, timeout):
+def kill_undead(tasks, timeout):
     """
     Signal all children that are over the given timeout.
     """
     for task in tasks:
         if timed_out(task, timeout):
             os.kill(task.pid, 9)
 
-def run_all_tests(tests, prefix, results, options):
+def run_all_tests_gen(tests, prefix, results, options):
     # Copy and reverse for fast pop off end.
     tests = tests[:]
     tests.reverse()
 
     # The set of currently running tests.
     tasks = []
 
     while len(tests) or len(tasks):
         while len(tests) and len(tasks) < options.worker_count:
             tasks.append(spawn_test(tests.pop(), prefix, options.passthrough))
 
-        timeout = get_max_wait(tasks, results, options.timeout)
+        timeout = get_max_wait(tasks, options.timeout)
         read_input(tasks, timeout)
 
-        kill_undead(tasks, results, options.timeout)
-        tasks = reap_zombies(tasks, results, options.timeout)
+        kill_undead(tasks, options.timeout)
+        tasks, finished = reap_zombies(tasks, options.timeout)
 
-        results.pb.poke()
+        # With Python3.4+ we could use yield from to remove this loop.
+        for out in finished:
+            yield out
 
     return True
+
+def run_all_tests(tests, prefix, results, options):
+    for out in run_all_tests_gen(tests, prefix, results, options):
+        results.push(out)
+    results.pb.poke()
+
+