Bug 1094624 - Resist to more multiprocessing problems in build/subconfigure.py. r=gps,a=kwierso
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 07 Nov 2014 10:16:42 +0900
changeset 240936 e3fac34f5ecb5d5ffe343923832a676010f6d97c
parent 240935 99aa737baeb9f6ece4593eff23d7ed363d6a3b47
child 240937 f129b17f906778c9b995a9f9d27682041ff14f9c
push id660
push userraliiev@mozilla.com
push dateWed, 18 Feb 2015 20:30:48 +0000
treeherdermozilla-release@49e493494178 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps, kwierso
bugs1094624
milestone36.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1094624 - Resist to more multiprocessing problems in build/subconfigure.py. r=gps,a=kwierso
build/subconfigure.py
--- a/build/subconfigure.py
+++ b/build/subconfigure.py
@@ -2,44 +2,43 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # This script is used to capture the content of config.status-generated
 # files and subsequently restore their timestamp if they haven't changed.
 
 import argparse
 import errno
+import itertools
 import os
 import re
 import subprocess
 import sys
 import pickle
 
 import mozpack.path as mozpath
 
-try:
-    from multiprocessing import Pool, cpu_count
-except ImportError:
-    import itertools
-
-    class Pool(object):
-        def __init__(self, size):
-            pass
 
-        def imap_unordered(self, fn, iterable):
-            return itertools.imap(fn, iterable)
-
-        def close(self):
-            pass
+class Pool(object):
+    def __new__(cls, size):
+        try:
+            import multiprocessing
+            size = min(size, multiprocessing.cpu_count())
+            return multiprocessing.Pool(size)
+        except:
+            return super(Pool, cls).__new__(cls)
 
-        def join(self):
-            pass
+    def imap_unordered(self, fn, iterable):
+        return itertools.imap(fn, iterable)
 
-    def cpu_count():
-        return 1
+    def close(self):
+        pass
+
+    def join(self):
+        pass
 
 
 class File(object):
     def __init__(self, path):
         self._path = path
         self._content = open(path, 'rb').read()
         stat = os.stat(path)
         self._times = (stat.st_atime, stat.st_mtime)
@@ -384,17 +383,17 @@ def subconfigure(args):
 
     if not subconfigures:
         return 0
 
     ret = 0
     # One would think using a ThreadPool would be faster, considering
     # everything happens in subprocesses anyways, but no, it's actually
     # slower on Windows. (20s difference overall!)
-    pool = Pool(min(len(subconfigures), cpu_count()))
+    pool = Pool(len(subconfigures))
     for relobjdir, returncode, output in \
             pool.imap_unordered(run, subconfigures):
         print prefix_lines(output, relobjdir)
         sys.stdout.flush()
         ret = max(returncode, ret)
         if ret:
             break
     pool.close()