bug 455578: create a post-upload script on stage to do release-to-dated, etc. - upload.py fixes and enhancements. r=ted
authorBen Hearsum <bhearsum@mozilla.com>
Wed, 24 Dec 2008 09:05:00 -0500
changeset 23041 36d425e44cff4398dfe6c5078b3cd3f31eb90c98
parent 22995 7127f726f66177c3bf21057933d44616b8f777af
child 23042 42d7cb939283be84e79740ce08b65aa40cd0e071
push idunknown
push userunknown
push dateunknown
reviewersted
bugs455578
milestone1.9.2a1pre
bug 455578: create a post-upload script on stage to do release-to-dated, etc. - upload.py fixes and enhancements. r=ted
build/upload.py
--- a/build/upload.py
+++ b/build/upload.py
@@ -52,16 +52,17 @@
 #
 # All files to be uploaded should be passed as commandline arguments to this
 # script. The script takes one other parameter, --base-path, which you can use
 # to indicate that files should be uploaded including their paths relative
 # to the base path.
 
 import sys, os
 from optparse import OptionParser
+from subprocess import Popen, PIPE
 from util import check_call
 
 def RequireEnvironmentVariable(v):
     """Return the value of the environment variable named v, or print
     an error and exit if it's unset (or empty)."""
     if not v in os.environ or os.environ[v] == "":
         print "Error: required environment variable %s not set" % v
         sys.exit(1)
@@ -99,25 +100,33 @@ def WindowsPathToMsysPath(path):
     return "/" + drive[0] + path.replace('\\','/')
 
 def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key):
     """Given optional port and ssh key values, append valid OpenSSH
     commandline arguments to the list cmdline if the values are not None."""
     if port is not None:
         cmdline.append("-P%d" % port)
     if ssh_key is not None:
-        cmdline.extend(["-i", WindowsPathToMsysPath(ssh_key)])
+        # Don't interpret ~ paths - ssh can handle that on its own
+        if not ssh_key.startswith('~'):
+            ssh_key = WindowsPathToMsysPath(ssh_key)
+        cmdline.extend(["-o", "IdentityFile=%s" % ssh_key])
 
 def DoSSHCommand(command, user, host, port=None, ssh_key=None):
     """Execute command on user@host using ssh. Optionally use
     port and ssh_key, if provided."""
     cmdline = ["ssh"]
     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
     cmdline.extend(["%s@%s" % (user, host), command])
-    check_call(cmdline)
+    cmd = Popen(cmdline, stdout=PIPE)
+    retcode = cmd.wait()
+    if retcode != 0:
+        raise Exception("Command %s returned non-zero exit code: %i" % \
+          (cmdline, retcode))
+    return cmd.stdout.read().strip()
 
 def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None):
     """Upload file to user@host:remote_path using scp. Optionally use
     port and ssh_key, if provided."""
     cmdline = ["scp"]
     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
     cmdline.extend([WindowsPathToMsysPath(file),
                     "%s@%s:%s" % (user, host, remote_path)])
@@ -130,23 +139,29 @@ def GetRemotePath(path, local_file, base
     the relative path from base_path to file."""
     if base_path is None or not local_file.startswith(base_path):
         return path
     dir = os.path.dirname(local_file)
     # strip base_path + extra slash and make it unixy
     dir = dir[len(base_path)+1:].replace('\\','/')
     return path + dir
 
-def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, post_upload_command=None):
+def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None):
     """Upload each file in the list files to user@host:path. Optionally pass
     port and ssh_key to the ssh commands. If base_path is not None, upload
-    files including their path relative to base_path. If post_upload_command
-    is not None, execute that command on the remote host after uploading
-    all files, passing it the upload path, and the full paths to all files
-    uploaded. If verbose is True, print status updates while working."""
+    files including their path relative to base_path. If upload_to_temp_dir is
+    True files will be uploaded to a temporary directory on the remote server.
+    Generally, you should have a post upload command specified in these cases
+    that can move them around to their correct location(s).
+    If post_upload_command is not None, execute that command on the remote host
+    after uploading all files, passing it the upload path, and the full paths to
+    all files uploaded.
+    If verbose is True, print status updates while working."""
+    if upload_to_temp_dir:
+        path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key)
     if not path.endswith("/"):
         path += "/"
     if base_path is not None:
         base_path = os.path.abspath(base_path)
     remote_files = []
     for file in files:
         file = os.path.abspath(file)
         if not os.path.isfile(file):
@@ -158,42 +173,50 @@ def UploadFiles(user, host, path, files,
             print "Uploading " + file
         DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
         remote_files.append(remote_path + '/' + os.path.basename(file))
     if post_upload_command is not None:
         if verbose:
             print "Running post-upload command: " + post_upload_command
         file_list = '"' + '" "'.join(remote_files) + '"'
         DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
+    if upload_to_temp_dir:
+        DoSSHCommand("rm -rf %s" % path, user, host, port=port, ssh_key=ssh_key)
     if verbose:
         print "Upload complete"
 
 if __name__ == '__main__':
     host = RequireEnvironmentVariable('UPLOAD_HOST')
     user = RequireEnvironmentVariable('UPLOAD_USER')
-    path = RequireEnvironmentVariable('UPLOAD_PATH')
+    path = OptionalEnvironmentVariable('UPLOAD_PATH')
+    upload_to_temp_dir = OptionalEnvironmentVariable('UPLOAD_TO_TEMP')
     port = OptionalEnvironmentVariable('UPLOAD_PORT')
     if port is not None:
         port = int(port)
     key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY')
     post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD')
+    if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir):
+        print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \
+              "defined."
+        sys.exit(1)
     if sys.platform == 'win32':
-        path = FixupMsysPath(path)
+        if path is not None:
+            path = FixupMsysPath(path)
         if post_upload_command is not None:
             post_upload_command = FixupMsysPath(post_upload_command)
 
     parser = OptionParser(usage="usage: %prog [options] <files>")
     parser.add_option("-b", "--base-path",
                       action="store", dest="base_path",
                       help="Preserve file paths relative to this path when uploading. If unset, all files will be uploaded directly to UPLOAD_PATH.")
     (options, args) = parser.parse_args()
     if len(args) < 1:
         print "You must specify at least one file to upload"
         sys.exit(1)
     try:
         UploadFiles(user, host, path, args, base_path=options.base_path,
-                    port=port, ssh_key=key, post_upload_command=post_upload_command,
+                    port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
+                    post_upload_command=post_upload_command,
                     verbose=True)
     except IOError, (strerror):
         print strerror
     except Exception, (err):
         print err
-