Bug 1307482 - Avoid excessive scp calls to make directories; r=ted
authorGregory Szorc <gps@mozilla.com>
Tue, 04 Oct 2016 08:34:30 -0700
changeset 361134 8764e36136539c656821bc3ae22d9ce3035679aa
parent 361133 7319e764beb990473e810660a2471aee9f065fe2
child 361135 1bcff6b5f303ee2eb74a8fe927c3dcc365463956
push id6795
push userjlund@mozilla.com
push dateMon, 23 Jan 2017 14:19:46 +0000
treeherdermozilla-beta@76101b503191 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1307482
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1307482 - Avoid excessive scp calls to make directories; r=ted Most files go to the same directory. This should cut down on the number of scp sessions we establish as part of directory creation. MozReview-Commit-ID: LpKwgUZhLEO
build/upload.py
--- a/build/upload.py
+++ b/build/upload.py
@@ -227,22 +227,32 @@ def UploadFiles(user, host, path, files,
         base_path = os.path.abspath(base_path)
     remote_files = []
     properties = {}
 
     def get_remote_path(p):
         return GetBaseRelativePath(path, os.path.abspath(p), base_path)
 
     try:
+        # Do a pass to find remote directories so we don't perform excessive
+        # scp calls.
+        remote_paths = set()
         for file in files:
             if not os.path.isfile(file):
                 raise IOError("File not found: %s" % file)
-            # first ensure that path exists remotely
+
+            remote_paths.add(get_remote_path(file))
+
+        # If we wanted to, we could reduce the remote paths if they are a parent
+        # of any entry.
+        for p in sorted(remote_paths):
+            DoSSHCommand("mkdir -p " + p, user, host, port=port, ssh_key=ssh_key)
+
+        for file in files:
             remote_path = get_remote_path(file)
-            DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
             if verbose:
                 print "Uploading " + file
             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
             remote_files.append(remote_path + '/' + os.path.basename(file))
         if post_upload_command is not None:
             if verbose:
                 print "Running post-upload command: " + post_upload_command
             file_list = '"' + '" "'.join(remote_files) + '"'