Bug 1198179 - make upload.py write properties even if not uploading; r=ted
authorMike Shal <mshal@mozilla.com>
Wed, 26 Aug 2015 09:46:08 -0400
changeset 294061 b41a88070b1ba516c9a25028a58a9d5da2ca17b5
parent 294060 36cd6ef44a0d39604767db4d8c29978be6adea01
child 294062 66b14ac1d547d17894caac7c3732580258faea8a
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1198179
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1198179 - make upload.py write properties even if not uploading; r=ted
build/upload.py
--- a/build/upload.py
+++ b/build/upload.py
@@ -3,17 +3,22 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 # When run directly, this script expects the following environment variables
 # to be set:
 # UPLOAD_HOST    : host to upload files to
 # UPLOAD_USER    : username on that host
+#  and one of the following:
 # UPLOAD_PATH    : path on that host to put the files in
+# UPLOAD_TO_TEMP : upload files to a new temporary directory
+#
+# If UPLOAD_HOST and UPLOAD_USER are not set, this script will simply write out
+# the properties file.
 #
 # And will use the following optional environment variables if set:
 # UPLOAD_SSH_KEY : path to a ssh private key to use
 # UPLOAD_PORT    : port to use for ssh
 # POST_UPLOAD_CMD: a commandline to run on the remote host after uploading.
 #                  UPLOAD_PATH and the full paths of all files uploaded will
 #                  be appended to the commandline.
 #
@@ -26,24 +31,16 @@ import sys, os
 import re
 import json
 import errno
 import hashlib
 from optparse import OptionParser
 from subprocess import check_call, check_output, STDOUT
 import redo
 
-def RequireEnvironmentVariable(v):
-    """Return the value of the environment variable named v, or print
-    an error and exit if it's unset (or empty)."""
-    if not v in os.environ or os.environ[v] == "":
-        print "Error: required environment variable %s not set" % v
-        sys.exit(1)
-    return os.environ[v]
-
 def OptionalEnvironmentVariable(v):
     """Return the value of the environment variable named v, or None
     if it's unset (or empty)."""
     if v in os.environ and os.environ[v] != "":
         return os.environ[v]
     return None
 
 def FixupMsysPath(path):
@@ -180,27 +177,34 @@ def GetUrlProperties(output, package):
                         properties.update({prop: m})
                         break
     except IOError as e:
         if e.errno != errno.ENOENT:
             raise
         properties = {prop: 'UNKNOWN' for prop, condition in property_conditions}
     return properties
 
-def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None, properties_file=None, package=None):
+def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None, package=None):
     """Upload each file in the list files to user@host:path. Optionally pass
     port and ssh_key to the ssh commands. If base_path is not None, upload
     files including their path relative to base_path. If upload_to_temp_dir is
     True files will be uploaded to a temporary directory on the remote server.
     Generally, you should have a post upload command specified in these cases
     that can move them around to their correct location(s).
     If post_upload_command is not None, execute that command on the remote host
     after uploading all files, passing it the upload path, and the full paths to
     all files uploaded.
     If verbose is True, print status updates while working."""
+    if not host or not user:
+        return {}
+    if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir):
+        print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \
+                "defined."
+        sys.exit(1)
+
     if upload_to_temp_dir:
         path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key)
     if not path.endswith("/"):
         path += "/"
     if base_path is not None:
         base_path = os.path.abspath(base_path)
     remote_files = []
     properties = {}
@@ -211,53 +215,53 @@ def UploadFiles(user, host, path, files,
                 raise IOError("File not found: %s" % file)
             # first ensure that path exists remotely
             remote_path = GetRemotePath(path, file, base_path)
             DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
             if verbose:
                 print "Uploading " + file
             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
             remote_files.append(remote_path + '/' + os.path.basename(file))
-
-            if file.endswith('.complete.mar'):
-                properties.update(GetMarProperties(file))
         if post_upload_command is not None:
             if verbose:
                 print "Running post-upload command: " + post_upload_command
             file_list = '"' + '" "'.join(remote_files) + '"'
             output = DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
             # We print since mozharness may parse URLs from the output stream.
             print output
-            if properties_file:
-                with open(properties_file, 'w') as outfile:
-                    properties.update(GetUrlProperties(output, package))
-                    properties['packageFilename'] = package
-                    properties['uploadFiles'] = [os.path.abspath(f) for f in files]
-                    json.dump(properties, outfile, indent=4)
+            properties = GetUrlProperties(output, package)
     finally:
         if upload_to_temp_dir:
             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
                          ssh_key=ssh_key)
     if verbose:
         print "Upload complete"
+    return properties
+
+def WriteProperties(files, properties_file, url_properties, package):
+    properties = url_properties
+    for file in files:
+        if file.endswith('.complete.mar'):
+            properties.update(GetMarProperties(file))
+    with open(properties_file, 'w') as outfile:
+        properties['packageFilename'] = package
+        properties['uploadFiles'] = [os.path.abspath(f) for f in files]
+        json.dump(properties, outfile, indent=4)
 
 if __name__ == '__main__':
-    host = RequireEnvironmentVariable('UPLOAD_HOST')
-    user = RequireEnvironmentVariable('UPLOAD_USER')
+    host = OptionalEnvironmentVariable('UPLOAD_HOST')
+    user = OptionalEnvironmentVariable('UPLOAD_USER')
     path = OptionalEnvironmentVariable('UPLOAD_PATH')
     upload_to_temp_dir = OptionalEnvironmentVariable('UPLOAD_TO_TEMP')
     port = OptionalEnvironmentVariable('UPLOAD_PORT')
     if port is not None:
         port = int(port)
     key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY')
     post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD')
-    if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir):
-        print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \
-              "defined."
-        sys.exit(1)
+
     if sys.platform == 'win32':
         if path is not None:
             path = FixupMsysPath(path)
         if post_upload_command is not None:
             post_upload_command = FixupMsysPath(post_upload_command)
 
     parser = OptionParser(usage="usage: %prog [options] <files>")
     parser.add_option("-b", "--base-path",
@@ -268,20 +272,24 @@ if __name__ == '__main__':
                       help="Path to the properties file to store the upload properties.")
     parser.add_option("--package",
                       action="store",
                       help="Name of the main package.")
     (options, args) = parser.parse_args()
     if len(args) < 1:
         print "You must specify at least one file to upload"
         sys.exit(1)
+    if not options.properties_file:
+        print "You must specify a --properties-file"
+        sys.exit(1)
     try:
-        UploadFiles(user, host, path, args, base_path=options.base_path,
-                    port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
-                    post_upload_command=post_upload_command,
-                    properties_file=options.properties_file, package=options.package,
-                    verbose=True)
+        url_properties = UploadFiles(user, host, path, args, base_path=options.base_path,
+                                     port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
+                                     post_upload_command=post_upload_command,
+                                     package=options.package,
+                                     verbose=True)
+        WriteProperties(args, options.properties_file, url_properties, options.package)
     except IOError, (strerror):
         print strerror
         sys.exit(1)
     except Exception, (err):
         print err
         sys.exit(2)