Bug 1118778 - Write upload properties from upload.py; r=glandium
authorMike Shal <mshal@mozilla.com>
Wed, 05 Aug 2015 15:43:15 -0400
changeset 290199 e15f2c2caaff757e8c3722cc9cfd5aae53f8a309
parent 290198 20c9570b07342a00d881cfb606695d1608626b16
child 290200 7f3739fddf0411ff53da8d11043b5dba53ed172e
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1118778
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1118778 - Write upload properties from upload.py; r=glandium
build/gen_mach_buildprops.py
build/moz-automation.mk
build/upload.py
toolkit/mozapps/installer/packager.mk
--- a/build/gen_mach_buildprops.py
+++ b/build/gen_mach_buildprops.py
@@ -35,92 +35,46 @@ def getMarProperties(filename, partial=F
     (mar_hash, mar_size) = getFileHashAndSize(filename)
     martype = 'partial' if partial else 'complete'
     return {
         '%sMarFilename' % martype: os.path.basename(filename),
         '%sMarSize' % martype: mar_size,
         '%sMarHash' % martype: mar_hash,
     }
 
-def getUrlProperties(filename, package):
-    # let's create a switch case using name-spaces/dict
-    # rather than a long if/else with duplicate code
-    property_conditions = [
-        # key: property name, value: condition
-        ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
-                       m.endswith('crashreporter-symbols-full.zip')),
-        ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
-        ('unsignedApkUrl', lambda m: m.endswith('apk') and
-                           'unsigned-unaligned' in m),
-        ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
-        ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
-        ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
-        ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
-        ('codeCoverageURL', lambda m: m.endswith('code-coverage-gcno.zip')),
-        ('sdkUrl', lambda m: m.endswith(('sdk.tar.bz2', 'sdk.zip'))),
-        ('testPackagesUrl', lambda m: m.endswith('test_packages.json')),
-        ('packageUrl', lambda m: m.endswith(package)),
-    ]
-    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz|json))$''')
-    properties = {}
-
-    try:
-        with open(filename) as f:
-            for line in f:
-                m = url_re.match(line)
-                if m:
-                    m = m.group(1)
-                    for prop, condition in property_conditions:
-                        if condition(m):
-                            properties.update({prop: m})
-                            break
-    except IOError as e:
-        if e.errno != errno.ENOENT:
-            raise
-        properties = {prop: 'UNKNOWN' for prop, condition in property_conditions}
-    return properties
-
 def getPartialInfo(props):
     return [{
         "from_buildid": props.get("previous_buildid"),
         "size": props.get("partialMarSize"),
         "hash": props.get("partialMarHash"),
         "url": props.get("partialMarUrl"),
     }]
 
 if __name__ == '__main__':
     parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
     parser.add_argument("--complete-mar-file", required=True,
                         action="store", dest="complete_mar_file",
                         help="Path to the complete MAR file, relative to the objdir.")
     parser.add_argument("--partial-mar-file", required=False,
                         action="store", dest="partial_mar_file",
                         help="Path to the partial MAR file, relative to the objdir.")
-    parser.add_argument("--upload-output", required=True,
-                        action="store", dest="upload_output",
-                        help="Path to the text output of 'make upload'")
-    parser.add_argument("--upload-files", required=True, nargs="+",
-                        action="store", dest="upload_files",
-                        help="List of files to be uploaded.")
-    parser.add_argument("--package", required=True,
-                        action="store", dest="package",
-                        help="Filename of the build package")
+    parser.add_argument("--upload-properties", required=True,
+                        action="store", dest="upload_properties",
+                        help="Path to the properties written by 'make upload'")
     args = parser.parse_args()
 
     json_data = getMarProperties(args.complete_mar_file)
-    json_data.update(getUrlProperties(args.upload_output, args.package))
+    with open(args.upload_properties) as f:
+        json_data.update(json.load(f))
     if args.partial_mar_file:
         json_data.update(getMarProperties(args.partial_mar_file, partial=True))
 
         # Pull the previous buildid from the partial mar filename.
         res = re.match(r'.*\.([0-9]+)-[0-9]+.mar', args.partial_mar_file)
         if res:
             json_data['previous_buildid'] = res.group(1)
 
             # Set partialInfo to be a collection of the partial mar properties
             # useful for balrog.
             json_data['partialInfo'] = getPartialInfo(json_data)
 
-    json_data['uploadFiles'] = args.upload_files
-    json_data['packageFilename'] = args.package
-
     with open('mach_build_properties.json', 'w') as outfile:
         json.dump(json_data, outfile, indent=4)
--- a/build/moz-automation.mk
+++ b/build/moz-automation.mk
@@ -12,19 +12,18 @@ endif
 endif
 
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 include $(topsrcdir)/toolkit/mozapps/installer/upload-files.mk
 
 # Clear out DIST_FILES if it was set by upload-files.mk (for Android builds)
 DIST_FILES =
 
-# Log file from the 'make upload' step. We need this to parse out the URLs of
-# the uploaded files.
-AUTOMATION_UPLOAD_OUTPUT = $(DIST)/automation-upload.txt
+# Properties from 'make upload' that are file URLs.
+AUTOMATION_UPLOAD_PROPERTIES = $(DIST)/upload-properties.json
 
 # Helper variables to convert from MOZ_AUTOMATION_* variables to the
 # corresponding the make target
 tier_MOZ_AUTOMATION_BUILD_SYMBOLS = buildsymbols
 tier_MOZ_AUTOMATION_L10N_CHECK = l10n-check
 tier_MOZ_AUTOMATION_PRETTY_L10N_CHECK = pretty-l10n-check
 tier_MOZ_AUTOMATION_INSTALLER = installer
 tier_MOZ_AUTOMATION_PRETTY_INSTALLER = pretty-installer
@@ -94,21 +93,17 @@ automation/sdk: automation/installer aut
 # conflicts in writing to the same files.
 automation/installer: automation/pretty-installer
 automation/package: automation/pretty-package
 automation/package-tests: automation/pretty-package-tests
 automation/l10n-check: automation/pretty-l10n-check
 automation/update-packaging: automation/pretty-update-packaging
 
 automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
-	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) $(addprefix --partial-mar-file ,$(wildcard $(DIST)/$(PARTIAL_MAR))) --upload-output $(AUTOMATION_UPLOAD_OUTPUT) --upload-files $(abspath $(UPLOAD_FILES)) --package $(PACKAGE)
-
-# We need the log from make upload to grep it for urls in order to set
-# properties.
-AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT)
+	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) $(addprefix --partial-mar-file ,$(wildcard $(DIST)/$(PARTIAL_MAR))) --upload-properties $(AUTOMATION_UPLOAD_PROPERTIES)
 
 # Note: We have to force -j1 here, at least until bug 1036563 is fixed.
 AUTOMATION_EXTRA_CMDLINE-l10n-check = -j1
 AUTOMATION_EXTRA_CMDLINE-pretty-l10n-check = -j1
 
 # And force -j1 here until bug 1077670 is fixed.
 AUTOMATION_EXTRA_CMDLINE-package-tests = -j1
 AUTOMATION_EXTRA_CMDLINE-pretty-package-tests = -j1
--- a/build/upload.py
+++ b/build/upload.py
@@ -18,18 +18,20 @@
 #                  be appended to the commandline.
 #
 # All files to be uploaded should be passed as commandline arguments to this
 # script. The script takes one other parameter, --base-path, which you can use
 # to indicate that files should be uploaded including their paths relative
 # to the base path.
 
 import sys, os
+import re
+import json
 from optparse import OptionParser
-from subprocess import check_call, check_output
+from subprocess import check_call, check_output, STDOUT
 import redo
 
 def RequireEnvironmentVariable(v):
     """Return the value of the environment variable named v, or print
     an error and exit if it's unset (or empty)."""
     if not v in os.environ or os.environ[v] == "":
         print "Error: required environment variable %s not set" % v
         sys.exit(1)
@@ -56,17 +58,19 @@ def FixupMsysPath(path):
             if path.startswith(msys):
                 path = path[len(msys):]
     return path
 
 def WindowsPathToMsysPath(path):
     """Translate a Windows pathname to an MSYS pathname.
     Necessary because we call out to ssh/scp, which are MSYS binaries
     and expect MSYS paths."""
-    if sys.platform != 'win32':
+    # If we're not on Windows, or if we already have an MSYS path (starting
+    # with '/' instead of 'c:' or something), then just return.
+    if sys.platform != 'win32' or path.startswith('/'):
         return path
     (drive, path) = os.path.splitdrive(os.path.abspath(path))
     return "/" + drive[0] + path.replace('\\','/')
 
 def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key):
     """Given optional port and ssh key values, append valid OpenSSH
     commandline arguments to the list cmdline if the values are not None."""
     if port is not None:
@@ -80,17 +84,17 @@ def AppendOptionalArgsToSSHCommandline(c
 def DoSSHCommand(command, user, host, port=None, ssh_key=None):
     """Execute command on user@host using ssh. Optionally use
     port and ssh_key, if provided."""
     cmdline = ["ssh"]
     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
     cmdline.extend(["%s@%s" % (user, host), command])
 
     with redo.retrying(check_output, sleeptime=10) as f:
-        output = f(cmdline).strip()
+        output = f(cmdline, stderr=STDOUT).strip()
         return output
 
     raise Exception("Command %s returned non-zero exit code" % cmdline)
 
 def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None):
     """Upload file to user@host:remote_path using scp. Optionally use
     port and ssh_key, if provided."""
     cmdline = ["scp"]
@@ -110,17 +114,54 @@ def GetRemotePath(path, local_file, base
     the relative path from base_path to file."""
     if base_path is None or not local_file.startswith(base_path):
         return path
     dir = os.path.dirname(local_file)
     # strip base_path + extra slash and make it unixy
     dir = dir[len(base_path)+1:].replace('\\','/')
     return path + dir
 
-def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None):
+def GetUrlProperties(output, package):
+    # let's create a switch case using name-spaces/dict
+    # rather than a long if/else with duplicate code
+    property_conditions = [
+        # key: property name, value: condition
+        ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
+                       m.endswith('crashreporter-symbols-full.zip')),
+        ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
+        ('unsignedApkUrl', lambda m: m.endswith('apk') and
+                           'unsigned-unaligned' in m),
+        ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
+        ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
+        ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
+        ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
+        ('codeCoverageURL', lambda m: m.endswith('code-coverage-gcno.zip')),
+        ('sdkUrl', lambda m: m.endswith(('sdk.tar.bz2', 'sdk.zip'))),
+        ('testPackagesUrl', lambda m: m.endswith('test_packages.json')),
+        ('packageUrl', lambda m: m.endswith(package)),
+    ]
+    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz|json))$''')
+    properties = {}
+
+    try:
+        for line in output.splitlines():
+            m = url_re.match(line.strip())
+            if m:
+                m = m.group(1)
+                for prop, condition in property_conditions:
+                    if condition(m):
+                        properties.update({prop: m})
+                        break
+    except IOError as e:
+        if e.errno != errno.ENOENT:
+            raise
+        properties = {prop: 'UNKNOWN' for prop, condition in property_conditions}
+    return properties
+
+def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None, properties_file=None, package=None):
     """Upload each file in the list files to user@host:path. Optionally pass
     port and ssh_key to the ssh commands. If base_path is not None, upload
     files including their path relative to base_path. If upload_to_temp_dir is
     True files will be uploaded to a temporary directory on the remote server.
     Generally, you should have a post upload command specified in these cases
     that can move them around to their correct location(s).
     If post_upload_command is not None, execute that command on the remote host
     after uploading all files, passing it the upload path, and the full paths to
@@ -144,17 +185,23 @@ def UploadFiles(user, host, path, files,
             if verbose:
                 print "Uploading " + file
             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
             remote_files.append(remote_path + '/' + os.path.basename(file))
         if post_upload_command is not None:
             if verbose:
                 print "Running post-upload command: " + post_upload_command
             file_list = '"' + '" "'.join(remote_files) + '"'
-            DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
+            output = DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
+            if properties_file:
+                with open(properties_file, 'w') as outfile:
+                    properties = GetUrlProperties(output, package)
+                    properties['packageFilename'] = package
+                    properties['uploadFiles'] = [os.path.abspath(f) for f in files]
+                    json.dump(properties, outfile, indent=4)
     finally:
         if upload_to_temp_dir:
             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
                          ssh_key=ssh_key)
     if verbose:
         print "Upload complete"
 
 if __name__ == '__main__':
@@ -174,25 +221,32 @@ if __name__ == '__main__':
     if sys.platform == 'win32':
         if path is not None:
             path = FixupMsysPath(path)
         if post_upload_command is not None:
             post_upload_command = FixupMsysPath(post_upload_command)
 
     parser = OptionParser(usage="usage: %prog [options] <files>")
     parser.add_option("-b", "--base-path",
-                      action="store", dest="base_path",
+                      action="store",
                       help="Preserve file paths relative to this path when uploading. If unset, all files will be uploaded directly to UPLOAD_PATH.")
+    parser.add_option("--properties-file",
+                      action="store",
+                      help="Path to the properties file to store the upload properties.")
+    parser.add_option("--package",
+                      action="store",
+                      help="Name of the main package.")
     (options, args) = parser.parse_args()
     if len(args) < 1:
         print "You must specify at least one file to upload"
         sys.exit(1)
     try:
         UploadFiles(user, host, path, args, base_path=options.base_path,
                     port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
                     post_upload_command=post_upload_command,
+                    properties_file=options.properties_file, package=options.package,
                     verbose=True)
     except IOError, (strerror):
         print strerror
         sys.exit(1)
     except Exception, (err):
         print err
         sys.exit(2)
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -203,16 +203,18 @@ checksum:
 	@echo 'CHECKSUM FILE START'
 	@cat $(CHECKSUM_FILE)
 	@echo 'CHECKSUM FILE END'
 	$(SIGN_CHECKSUM_CMD)
 
 
 upload: checksum
 	$(PYTHON) $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) \
+		--package $(PACKAGE) \
+		--properties-file $(DIST)/upload-properties.json \
 		$(UPLOAD_FILES) \
 		$(CHECKSUM_FILES)
 
 # source-package creates a source tarball from the files in MOZ_PKG_SRCDIR,
 # which is either set to a clean checkout or defaults to $topsrcdir
 source-package:
 	@echo 'Packaging source tarball...'
 	$(MKDIR) -p $(DIST)/$(PKG_SRCPACK_PATH)