Bug 1198179 - Kill gen_mach_buildprops.py; r=ted
authorMike Shal <mshal@mozilla.com>
Tue, 25 Aug 2015 11:42:44 -0400
changeset 261476 36cd6ef44a0d39604767db4d8c29978be6adea01
parent 261475 873acda091b75ba8aee7f374cd8e6c8627d64494
child 261477 b41a88070b1ba516c9a25028a58a9d5da2ca17b5
push id17395
push usercbook@mozilla.com
push dateWed, 09 Sep 2015 12:16:20 +0000
treeherderb2g-inbound@e21caed7a7f4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1198179
milestone43.0a1
Bug 1198179 - Kill gen_mach_buildprops.py; r=ted
build/gen_mach_buildprops.py
build/moz-automation.mk
build/upload.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
toolkit/mozapps/installer/packager.mk
deleted file mode 100644
--- a/build/gen_mach_buildprops.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/python
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import sys
-import os
-import hashlib
-import json
-import re
-import errno
-from argparse import ArgumentParser
-
-def getFileHashAndSize(filename):
-    sha512Hash = 'UNKNOWN'
-    size = 'UNKNOWN'
-
-    try:
-        # open in binary mode to make sure we get consistent results
-        # across all platforms
-        f = open(filename, "rb")
-        shaObj = hashlib.sha512(f.read())
-        sha512Hash = shaObj.hexdigest()
-
-        size = os.path.getsize(filename)
-    except:
-        pass
-
-    return (sha512Hash, size)
-
-def getMarProperties(filename, partial=False):
-    if not os.path.exists(filename):
-        return {}
-    (mar_hash, mar_size) = getFileHashAndSize(filename)
-    martype = 'partial' if partial else 'complete'
-    return {
-        '%sMarFilename' % martype: os.path.basename(filename),
-        '%sMarSize' % martype: mar_size,
-        '%sMarHash' % martype: mar_hash,
-    }
-
-def getPartialInfo(props):
-    return [{
-        "from_buildid": props.get("previous_buildid"),
-        "size": props.get("partialMarSize"),
-        "hash": props.get("partialMarHash"),
-        "url": props.get("partialMarUrl"),
-    }]
-
-if __name__ == '__main__':
-    parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
-    parser.add_argument("--complete-mar-file", required=True,
-                        action="store", dest="complete_mar_file",
-                        help="Path to the complete MAR file, relative to the objdir.")
-    parser.add_argument("--partial-mar-file", required=False,
-                        action="store", dest="partial_mar_file",
-                        help="Path to the partial MAR file, relative to the objdir.")
-    parser.add_argument("--upload-properties", required=False,
-                        action="store", dest="upload_properties",
-                        help="Path to the properties written by 'make upload'")
-    args = parser.parse_args()
-
-    json_data = getMarProperties(args.complete_mar_file)
-    if args.upload_properties:
-        with open(args.upload_properties) as f:
-            json_data.update(json.load(f))
-    if args.partial_mar_file:
-        json_data.update(getMarProperties(args.partial_mar_file, partial=True))
-
-        # Pull the previous buildid from the partial mar filename.
-        res = re.match(r'.*\.([0-9]+)-[0-9]+.mar', args.partial_mar_file)
-        if res:
-            json_data['previous_buildid'] = res.group(1)
-
-            # Set partialInfo to be a collection of the partial mar properties
-            # useful for balrog.
-            json_data['partialInfo'] = getPartialInfo(json_data)
-
-    with open('mach_build_properties.json', 'w') as outfile:
-        json.dump(json_data, outfile, indent=4)
--- a/build/moz-automation.mk
+++ b/build/moz-automation.mk
@@ -12,21 +12,16 @@ endif
 endif
 
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 include $(topsrcdir)/toolkit/mozapps/installer/upload-files.mk
 
 # Clear out DIST_FILES if it was set by upload-files.mk (for Android builds)
 DIST_FILES =
 
-ifeq (1,$(MOZ_AUTOMATION_UPLOAD))
-# Properties from 'make upload' that are file URLs.
-AUTOMATION_UPLOAD_PROPERTIES = --upload-properties $(DIST)/upload-properties.json
-endif
-
 # Helper variables to convert from MOZ_AUTOMATION_* variables to the
 # corresponding the make target
 tier_MOZ_AUTOMATION_BUILD_SYMBOLS = buildsymbols
 tier_MOZ_AUTOMATION_L10N_CHECK = l10n-check
 tier_MOZ_AUTOMATION_PRETTY_L10N_CHECK = pretty-l10n-check
 tier_MOZ_AUTOMATION_INSTALLER = installer
 tier_MOZ_AUTOMATION_PRETTY_INSTALLER = pretty-installer
 tier_MOZ_AUTOMATION_PACKAGE = package
@@ -95,17 +90,17 @@ automation/sdk: automation/installer aut
 # conflicts in writing to the same files.
 automation/installer: automation/pretty-installer
 automation/package: automation/pretty-package
 automation/package-tests: automation/pretty-package-tests
 automation/l10n-check: automation/pretty-l10n-check
 automation/update-packaging: automation/pretty-update-packaging
 
 automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
-	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) $(addprefix --partial-mar-file ,$(wildcard $(DIST)/$(PARTIAL_MAR))) $(AUTOMATION_UPLOAD_PROPERTIES)
+	@echo Automation steps completed.
 
 # Note: We have to force -j1 here, at least until bug 1036563 is fixed.
 AUTOMATION_EXTRA_CMDLINE-l10n-check = -j1
 AUTOMATION_EXTRA_CMDLINE-pretty-l10n-check = -j1
 
 # And force -j1 here until bug 1077670 is fixed.
 AUTOMATION_EXTRA_CMDLINE-package-tests = -j1
 AUTOMATION_EXTRA_CMDLINE-pretty-package-tests = -j1
--- a/build/upload.py
+++ b/build/upload.py
@@ -20,16 +20,18 @@
 # All files to be uploaded should be passed as commandline arguments to this
 # script. The script takes one other parameter, --base-path, which you can use
 # to indicate that files should be uploaded including their paths relative
 # to the base path.
 
 import sys, os
 import re
 import json
+import errno
+import hashlib
 from optparse import OptionParser
 from subprocess import check_call, check_output, STDOUT
 import redo
 
 def RequireEnvironmentVariable(v):
     """Return the value of the environment variable named v, or print
     an error and exit if it's unset (or empty)."""
     if not v in os.environ or os.environ[v] == "":
@@ -114,16 +116,43 @@ def GetRemotePath(path, local_file, base
     the relative path from base_path to file."""
     if base_path is None or not local_file.startswith(base_path):
         return path
     dir = os.path.dirname(local_file)
     # strip base_path + extra slash and make it unixy
     dir = dir[len(base_path)+1:].replace('\\','/')
     return path + dir
 
+def GetFileHashAndSize(filename):
+    sha512Hash = 'UNKNOWN'
+    size = 'UNKNOWN'
+
+    try:
+        # open in binary mode to make sure we get consistent results
+        # across all platforms
+        with open(filename, "rb") as f:
+            shaObj = hashlib.sha512(f.read())
+            sha512Hash = shaObj.hexdigest()
+
+        size = os.path.getsize(filename)
+    except:
+        raise Exception("Unable to get filesize/hash from file: %s" % filename)
+
+    return (sha512Hash, size)
+
+def GetMarProperties(filename):
+    if not os.path.exists(filename):
+        return {}
+    (mar_hash, mar_size) = GetFileHashAndSize(filename)
+    return {
+        'completeMarFilename': os.path.basename(filename),
+        'completeMarSize': mar_size,
+        'completeMarHash': mar_hash,
+    }
+
 def GetUrlProperties(output, package):
     # let's create a switch case using name-spaces/dict
     # rather than a long if/else with duplicate code
     property_conditions = [
         # key: property name, value: condition
         ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
                        m.endswith('crashreporter-symbols-full.zip')),
         ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
@@ -169,38 +198,42 @@ def UploadFiles(user, host, path, files,
     If verbose is True, print status updates while working."""
     if upload_to_temp_dir:
         path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key)
     if not path.endswith("/"):
         path += "/"
     if base_path is not None:
         base_path = os.path.abspath(base_path)
     remote_files = []
+    properties = {}
     try:
         for file in files:
             file = os.path.abspath(file)
             if not os.path.isfile(file):
                 raise IOError("File not found: %s" % file)
             # first ensure that path exists remotely
             remote_path = GetRemotePath(path, file, base_path)
             DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
             if verbose:
                 print "Uploading " + file
             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
             remote_files.append(remote_path + '/' + os.path.basename(file))
+
+            if file.endswith('.complete.mar'):
+                properties.update(GetMarProperties(file))
         if post_upload_command is not None:
             if verbose:
                 print "Running post-upload command: " + post_upload_command
             file_list = '"' + '" "'.join(remote_files) + '"'
             output = DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
             # We print since mozharness may parse URLs from the output stream.
             print output
             if properties_file:
                 with open(properties_file, 'w') as outfile:
-                    properties = GetUrlProperties(output, package)
+                    properties.update(GetUrlProperties(output, package))
                     properties['packageFilename'] = package
                     properties['uploadFiles'] = [os.path.abspath(f) for f in files]
                     json.dump(properties, outfile, indent=4)
     finally:
         if upload_to_temp_dir:
             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
                          ssh_key=ssh_key)
     if verbose:
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1236,17 +1236,17 @@ or run without that action (ie: --no-{ac
         else:
             # capitalize every word in between '-'
             branch_list = self.branch.split('-')
             branch_list = [elem.capitalize() for elem in branch_list]
             return '-'.join(branch_list)
 
     def _query_props_set_by_mach(self, console_output=True, error_level=FATAL):
         mach_properties_path = os.path.join(
-            self.query_abs_dirs()['abs_obj_dir'], 'mach_build_properties.json'
+            self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json'
         )
         self.info("setting properties set by mach build. Looking in path: %s"
                   % mach_properties_path)
         if os.path.exists(mach_properties_path):
             with self.opened(mach_properties_path, error_level=error_level) as (fh, err):
                 build_props = json.load(fh)
                 if err:
                     self.log("%s exists but there was an error reading the "
@@ -1257,19 +1257,17 @@ or run without that action (ie: --no-{ac
                              error_level)
                 if console_output:
                     self.info("Properties set from 'mach build'")
                     self.info(pprint.pformat(build_props))
             for key, prop in build_props.iteritems():
                 if prop != 'UNKNOWN':
                     self.set_buildbot_property(key, prop, write_to_file=True)
         else:
-            self.log("Could not determine path for build properties. "
-                     "Does this exist: `%s` ?" % mach_properties_path,
-                     level=error_level)
+            self.info("No mach_build_properties.json found - not importing properties.")
 
     def generate_build_props(self, console_output=True, halt_on_failure=False):
         """sets props found from mach build and, in addition, buildid,
         sourcestamp,  appVersion, and appName."""
 
         error_level = ERROR
         if halt_on_failure:
             error_level = FATAL
@@ -1639,17 +1637,17 @@ or run without that action (ie: --no-{ac
             # the old package should live in source dir so we don't need to do
             # this for nighties since we clobber the whole work_dir in
             # clobber()
             self._rm_old_package()
         self._get_mozconfig()
         self._run_tooltool()
         self._create_mozbuild_dir()
         mach_props = os.path.join(
-            self.query_abs_dirs()['abs_obj_dir'], 'mach_build_properties.json'
+            self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json'
         )
         if os.path.exists(mach_props):
             self.info("Removing previous mach property file: %s" % mach_props)
             self.rmtree(mach_props)
 
     def build(self):
         """builds application."""
         env = self.query_build_env()
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -204,17 +204,17 @@ checksum:
 	@cat $(CHECKSUM_FILE)
 	@echo 'CHECKSUM FILE END'
 	$(SIGN_CHECKSUM_CMD)
 
 
 upload: checksum
 	$(PYTHON) $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) \
 		--package $(PACKAGE) \
-		--properties-file $(DIST)/upload-properties.json \
+		--properties-file $(DIST)/mach_build_properties.json \
 		$(UPLOAD_FILES) \
 		$(CHECKSUM_FILES)
 
 # source-package creates a source tarball from the files in MOZ_PKG_SRCDIR,
 # which is either set to a clean checkout or defaults to $topsrcdir
 source-package:
 	@echo 'Packaging source tarball...'
 	$(MKDIR) -p $(DIST)/$(PKG_SRCPACK_PATH)