Bug 1529691 - use futures in order to process in parallel clang-format. r=bbouvier
authorAndi-Bogdan Postelnicu <bpostelnicu@mozilla.com>
Tue, 12 Mar 2019 18:58:46 +0000
changeset 521585 add98afa5f0ca1ca17aa87eaa045442a3e1fa07a
parent 521584 01aa96181b7b4cee86095a96e4f4d38d1b8a479a
child 521586 a001f492200e62cf914374eebcde837f81f9e8e7
child 521622 1994e1ce54af5d74d19eae64743f4e4d82e80db1
child 521679 47fef0eb16e990e80375352a937c07160a27ef28
push id10867
push userdvarga@mozilla.com
push dateThu, 14 Mar 2019 15:20:45 +0000
treeherdermozilla-beta@abad13547875 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbbouvier
bugs1529691, 1521772
milestone67.0a1
first release with
nightly linux32
add98afa5f0c / 67.0a1 / 20190312214352 / files
nightly linux64
add98afa5f0c / 67.0a1 / 20190312214352 / files
nightly mac
add98afa5f0c / 67.0a1 / 20190312214352 / files
nightly win32
add98afa5f0c / 67.0a1 / 20190312214352 / files
nightly win64
add98afa5f0c / 67.0a1 / 20190312214352 / files
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
releases
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1529691 - use futures in order to process in parallel clang-format. r=bbouvier Parts of this patch were taken from the original work of :bbouvier in Bug 1521772. We needed to revert Bug 1521772 since it broken Windows compatibility. Differential Revision: https://phabricator.services.mozilla.com/D23100
python/mozbuild/mozbuild/mach_commands.py
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -70,16 +70,25 @@ If you feel this message is not appropri
 please file a Firefox Build System :: General bug at
 https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=General
 and tell us about your machine and build configuration so we can adjust the
 warning heuristic.
 ===================
 '''
 
 
+# Function used to run clang-format on a batch of files. It is a helper function
+# in order to integrate into the futures ecosystem clang-format.
+def run_one_clang_format_batch(args):
+    try:
+        subprocess.check_output(args)
+    except subprocess.CalledProcessError as e:
+        return e
+
+
 class StoreDebugParamsAndWarnAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         sys.stderr.write('The --debugparams argument is deprecated. Please ' +
                          'use --debugger-args instead.\n\n')
         setattr(namespace, self.dest, values)
 
 
 @CommandProvider
@@ -2866,56 +2875,81 @@ class StaticAnalysis(MachCommandBase):
 
         path_list = self._generate_path_list(paths)
 
         if path_list == []:
             return
 
         print("Processing %d file(s)..." % len(path_list))
 
-        batchsize = 200
         if show:
-            batchsize = 1
-
-        for i in range(0, len(path_list), batchsize):
-            l = path_list[i: (i + batchsize)]
-            if show:
+            for i in range(0, len(path_list)):
+                l = path_list[i: (i + 1)]
+
                 # Copy the files into a temp directory
                 # and run clang-format on the temp directory
                 # and show the diff
                 original_path = l[0]
                 local_path = ntpath.basename(original_path)
                 target_file = os.path.join(tmpdir, local_path)
                 faketmpdir = os.path.dirname(target_file)
                 if not os.path.isdir(faketmpdir):
                     os.makedirs(faketmpdir)
                 shutil.copy(l[0], faketmpdir)
                 l[0] = target_file
 
-            # Run clang-format on the list
-            try:
-                check_output(args + l)
-            except CalledProcessError as e:
-                # Something wrong happend
-                print("clang-format: An error occured while running clang-format.")
-                return e.returncode
-
-            if show:
+                # Run clang-format on the list
+                try:
+                    check_output(args + l)
+                except CalledProcessError as e:
+                    # Something wrong happend
+                    print("clang-format: An error occured while running clang-format.")
+                    return e.returncode
+
                 # show the diff
                 diff_command = ["diff", "-u", original_path, target_file]
                 try:
                     output = check_output(diff_command)
                 except CalledProcessError as e:
                     # diff -u returns 0 when no change
                     # here, we expect changes. if we are here, this means that
                     # there is a diff to show
                     if e.output:
                         print(e.output)
-        if show:
+
             shutil.rmtree(tmpdir)
+            return 0
+
+        # Run clang-format in parallel trying to saturate all of the available cores.
+        import concurrent.futures
+        import multiprocessing
+        import math
+
+        max_workers = multiprocessing.cpu_count()
+        batchsize = int(math.ceil(float(len(path_list)) / max_workers))
+
+        batches = []
+        for i in range(0, len(path_list), batchsize):
+            batches.append(args + path_list[i: (i + batchsize)])
+
+        error_code = None
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = []
+            for batch in batches:
+                futures.append(executor.submit(run_one_clang_format_batch, batch))
+
+            for future in concurrent.futures.as_completed(futures):
+                # Wait for every task to finish
+                ret_val = future.result()
+                if ret_val is not None:
+                    error_code = ret_val
+
+            if error_code is not None:
+                return error_code
         return 0
 
 @CommandProvider
 class Vendor(MachCommandBase):
     """Vendor third-party dependencies into the source repository."""
 
     @Command('vendor', category='misc',
              description='Vendor third-party dependencies into the source repository.')