Bug 974281 - Do not clobber msvc directory by default; r?ted draft
authorGregory Szorc <gps@mozilla.com>
Fri, 11 Mar 2016 15:11:33 -0800
changeset 394046 71ef2adbb931e2d7a9068a11d27a2b04a6e2012c
parent 394045 df392e8832e52501950f09acda208943c9e3d707
child 526719 832041eabc0534a9f525666b2390ddbf47d935bf
push id24468
push userbmo:gps@mozilla.com
push dateFri, 29 Jul 2016 00:03:24 +0000
reviewersted
bugs974281
milestone50.0a1
Bug 974281 - Do not clobber msvc directory by default; r?ted The "remove_objdir" method has been rewritten to support partial clobber. It still defaults to full clobber. But the "full" argument can be passed as False to limit to a partial clobber where currently the "msvc" directory (contains Visual Studio files) is not clobbered. On Windows, there might be a regression with this change because we'll be invoking N winrm.exe processes and new processes have a non-trivial overhead on Windows. However, it is hopefully unlikely that new processes are more overhead than deleting hundreds of thousands of files. MozReview-Commit-ID: 7yeMttztwic
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/mach_commands.py
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import errno
 import json
 import logging
 import mozpack.path as mozpath
 import multiprocessing
 import os
 import subprocess
 import sys
 import which
@@ -320,25 +321,53 @@ class MozbuildObject(ProcessExecutionMix
         try:
             p = subprocess.Popen(['winrm.exe', '-h'],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)
             return p.wait() == 1 and p.stdout.read().startswith('winrm')
         except:
             return False
 
-    def remove_objdir(self):
-        """Remove the entire object directory."""
+    def remove_objdir(self, full=True):
+        """Remove the object directory.
+
+        ``full`` controls whether to fully delete the objdir. If False,
+        some directories (e.g. Visual Studio Project Files) will not be
+        deleted.
+        """
+        # Top-level files and directories to not clobber by default.
+        no_clobber = {
+            'msvc',
+        }
 
-        if sys.platform.startswith('win') and self.have_winrm():
-            subprocess.check_call(['winrm', '-rf', self.topobjdir])
+        if full:
+            # mozfile doesn't like unicode arguments (bug 818783).
+            paths = [self.topobjdir.encode('utf-8')]
         else:
-            # We use mozfile because it is faster than shutil.rmtree().
-            # mozfile doesn't like unicode arguments (bug 818783).
-            mozfileremove(self.topobjdir.encode('utf-8'))
+            try:
+                paths = []
+                for p in os.listdir(self.topobjdir):
+                    if p not in no_clobber:
+                        paths.append(os.path.join(self.topobjdir, p).encode('utf-8'))
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                return
+
+        procs = []
+        for p in sorted(paths):
+            path = os.path.join(self.topobjdir, p)
+            if sys.platform.startswith('win') and self.have_winrm() and os.path.isdir(path):
+                procs.append(subprocess.Popen(['winrm', '-rf', path]))
+            else:
+                # We use mozfile because it is faster than shutil.rmtree().
+                mozfileremove(path)
+
+        for p in procs:
+            p.wait()
 
     def get_binary_path(self, what='app', validate_exists=True, where='default'):
         """Obtain the path to a compiled binary for this build configuration.
 
         The what argument is the program or tool being sought after. See the
         code implementation for supported values.
 
         If validate_exists is True (the default), we will ensure the found path
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -626,26 +626,28 @@ class Doctor(MachCommandBase):
 class Clobber(MachCommandBase):
     NO_AUTO_LOG = True
     CLOBBER_CHOICES = ['objdir', 'python']
     @Command('clobber', category='build',
         description='Clobber the tree (delete the object directory).')
     @CommandArgument('what', default=['objdir'], nargs='*',
         help='Target to clobber, must be one of {{{}}} (default objdir).'.format(
              ', '.join(CLOBBER_CHOICES)))
-    def clobber(self, what):
+    @CommandArgument('--full', action='store_true',
+        help='Perform a full clobber')
+    def clobber(self, what, full=False):
         invalid = set(what) - set(self.CLOBBER_CHOICES)
         if invalid:
             print('Unknown clobber target(s): {}'.format(', '.join(invalid)))
             return 1
 
         ret = 0
         if 'objdir' in what:
             try:
-                self.remove_objdir()
+                self.remove_objdir(full=full)
             except OSError as e:
                 if sys.platform.startswith('win'):
                     if isinstance(e, WindowsError) and e.winerror in (5,32):
                         self.log(logging.ERROR, 'file_access_error', {'error': e},
                             "Could not clobber because a file was in use. If the "
                             "application is running, try closing it. {error}")
                         return 1
                 raise