Bug 1198179 - Kill gen_mach_buildprops.py; r=ted
authorMike Shal <mshal@mozilla.com>
Tue, 25 Aug 2015 11:42:44 -0400
changeset 294060 36cd6ef44a0d39604767db4d8c29978be6adea01
parent 294059 873acda091b75ba8aee7f374cd8e6c8627d64494
child 294061 b41a88070b1ba516c9a25028a58a9d5da2ca17b5
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1198179
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1198179 - Kill gen_mach_buildprops.py; r=ted
build/gen_mach_buildprops.py
build/moz-automation.mk
build/upload.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
toolkit/mozapps/installer/packager.mk
deleted file mode 100644
--- a/build/gen_mach_buildprops.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/python
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import sys
-import os
-import hashlib
-import json
-import re
-import errno
-from argparse import ArgumentParser
-
-def getFileHashAndSize(filename):
-    sha512Hash = 'UNKNOWN'
-    size = 'UNKNOWN'
-
-    try:
-        # open in binary mode to make sure we get consistent results
-        # across all platforms
-        f = open(filename, "rb")
-        shaObj = hashlib.sha512(f.read())
-        sha512Hash = shaObj.hexdigest()
-
-        size = os.path.getsize(filename)
-    except:
-        pass
-
-    return (sha512Hash, size)
-
-def getMarProperties(filename, partial=False):
-    if not os.path.exists(filename):
-        return {}
-    (mar_hash, mar_size) = getFileHashAndSize(filename)
-    martype = 'partial' if partial else 'complete'
-    return {
-        '%sMarFilename' % martype: os.path.basename(filename),
-        '%sMarSize' % martype: mar_size,
-        '%sMarHash' % martype: mar_hash,
-    }
-
-def getPartialInfo(props):
-    return [{
-        "from_buildid": props.get("previous_buildid"),
-        "size": props.get("partialMarSize"),
-        "hash": props.get("partialMarHash"),
-        "url": props.get("partialMarUrl"),
-    }]
-
-if __name__ == '__main__':
-    parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
-    parser.add_argument("--complete-mar-file", required=True,
-                        action="store", dest="complete_mar_file",
-                        help="Path to the complete MAR file, relative to the objdir.")
-    parser.add_argument("--partial-mar-file", required=False,
-                        action="store", dest="partial_mar_file",
-                        help="Path to the partial MAR file, relative to the objdir.")
-    parser.add_argument("--upload-properties", required=False,
-                        action="store", dest="upload_properties",
-                        help="Path to the properties written by 'make upload'")
-    args = parser.parse_args()
-
-    json_data = getMarProperties(args.complete_mar_file)
-    if args.upload_properties:
-        with open(args.upload_properties) as f:
-            json_data.update(json.load(f))
-    if args.partial_mar_file:
-        json_data.update(getMarProperties(args.partial_mar_file, partial=True))
-
-        # Pull the previous buildid from the partial mar filename.
-        res = re.match(r'.*\.([0-9]+)-[0-9]+.mar', args.partial_mar_file)
-        if res:
-            json_data['previous_buildid'] = res.group(1)
-
-            # Set partialInfo to be a collection of the partial mar properties
-            # useful for balrog.
-            json_data['partialInfo'] = getPartialInfo(json_data)
-
-    with open('mach_build_properties.json', 'w') as outfile:
-        json.dump(json_data, outfile, indent=4)
--- a/build/moz-automation.mk
+++ b/build/moz-automation.mk
@@ -12,21 +12,16 @@ endif
 endif
 
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 include $(topsrcdir)/toolkit/mozapps/installer/upload-files.mk
 
 # Clear out DIST_FILES if it was set by upload-files.mk (for Android builds)
 DIST_FILES =
 
-ifeq (1,$(MOZ_AUTOMATION_UPLOAD))
-# Properties from 'make upload' that are file URLs.
-AUTOMATION_UPLOAD_PROPERTIES = --upload-properties $(DIST)/upload-properties.json
-endif
-
 # Helper variables to convert from MOZ_AUTOMATION_* variables to the
 # corresponding the make target
 tier_MOZ_AUTOMATION_BUILD_SYMBOLS = buildsymbols
 tier_MOZ_AUTOMATION_L10N_CHECK = l10n-check
 tier_MOZ_AUTOMATION_PRETTY_L10N_CHECK = pretty-l10n-check
 tier_MOZ_AUTOMATION_INSTALLER = installer
 tier_MOZ_AUTOMATION_PRETTY_INSTALLER = pretty-installer
 tier_MOZ_AUTOMATION_PACKAGE = package
@@ -95,17 +90,17 @@ automation/sdk: automation/installer aut
 # conflicts in writing to the same files.
 automation/installer: automation/pretty-installer
 automation/package: automation/pretty-package
 automation/package-tests: automation/pretty-package-tests
 automation/l10n-check: automation/pretty-l10n-check
 automation/update-packaging: automation/pretty-update-packaging
 
 automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
-	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) $(addprefix --partial-mar-file ,$(wildcard $(DIST)/$(PARTIAL_MAR))) $(AUTOMATION_UPLOAD_PROPERTIES)
+	@echo Automation steps completed.
 
 # Note: We have to force -j1 here, at least until bug 1036563 is fixed.
 AUTOMATION_EXTRA_CMDLINE-l10n-check = -j1
 AUTOMATION_EXTRA_CMDLINE-pretty-l10n-check = -j1
 
 # And force -j1 here until bug 1077670 is fixed.
 AUTOMATION_EXTRA_CMDLINE-package-tests = -j1
 AUTOMATION_EXTRA_CMDLINE-pretty-package-tests = -j1
--- a/build/upload.py
+++ b/build/upload.py
@@ -20,16 +20,18 @@
 # All files to be uploaded should be passed as commandline arguments to this
 # script. The script takes one other parameter, --base-path, which you can use
 # to indicate that files should be uploaded including their paths relative
 # to the base path.
 
 import sys, os
 import re
 import json
+import errno
+import hashlib
 from optparse import OptionParser
 from subprocess import check_call, check_output, STDOUT
 import redo
 
 def RequireEnvironmentVariable(v):
     """Return the value of the environment variable named v, or print
     an error and exit if it's unset (or empty)."""
     if not v in os.environ or os.environ[v] == "":
@@ -114,16 +116,43 @@ def GetRemotePath(path, local_file, base
     the relative path from base_path to file."""
     if base_path is None or not local_file.startswith(base_path):
         return path
     dir = os.path.dirname(local_file)
     # strip base_path + extra slash and make it unixy
     dir = dir[len(base_path)+1:].replace('\\','/')
     return path + dir
 
+def GetFileHashAndSize(filename):
+    sha512Hash = 'UNKNOWN'
+    size = 'UNKNOWN'
+
+    try:
+        # open in binary mode to make sure we get consistent results
+        # across all platforms
+        with open(filename, "rb") as f:
+            shaObj = hashlib.sha512(f.read())
+            sha512Hash = shaObj.hexdigest()
+
+        size = os.path.getsize(filename)
+    except:
+        raise Exception("Unable to get filesize/hash from file: %s" % filename)
+
+    return (sha512Hash, size)
+
+def GetMarProperties(filename):
+    if not os.path.exists(filename):
+        return {}
+    (mar_hash, mar_size) = GetFileHashAndSize(filename)
+    return {
+        'completeMarFilename': os.path.basename(filename),
+        'completeMarSize': mar_size,
+        'completeMarHash': mar_hash,
+    }
+
 def GetUrlProperties(output, package):
     # let's create a switch case using name-spaces/dict
     # rather than a long if/else with duplicate code
     property_conditions = [
         # key: property name, value: condition
         ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
                        m.endswith('crashreporter-symbols-full.zip')),
         ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
@@ -169,38 +198,42 @@ def UploadFiles(user, host, path, files,
     If verbose is True, print status updates while working."""
     if upload_to_temp_dir:
         path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key)
     if not path.endswith("/"):
         path += "/"
     if base_path is not None:
         base_path = os.path.abspath(base_path)
     remote_files = []
+    properties = {}
     try:
         for file in files:
             file = os.path.abspath(file)
             if not os.path.isfile(file):
                 raise IOError("File not found: %s" % file)
             # first ensure that path exists remotely
             remote_path = GetRemotePath(path, file, base_path)
             DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
             if verbose:
                 print "Uploading " + file
             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
             remote_files.append(remote_path + '/' + os.path.basename(file))
+
+            if file.endswith('.complete.mar'):
+                properties.update(GetMarProperties(file))
         if post_upload_command is not None:
             if verbose:
                 print "Running post-upload command: " + post_upload_command
             file_list = '"' + '" "'.join(remote_files) + '"'
             output = DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
             # We print since mozharness may parse URLs from the output stream.
             print output
             if properties_file:
                 with open(properties_file, 'w') as outfile:
-                    properties = GetUrlProperties(output, package)
+                    properties.update(GetUrlProperties(output, package))
                     properties['packageFilename'] = package
                     properties['uploadFiles'] = [os.path.abspath(f) for f in files]
                     json.dump(properties, outfile, indent=4)
     finally:
         if upload_to_temp_dir:
             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
                          ssh_key=ssh_key)
     if verbose:
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1236,17 +1236,17 @@ or run without that action (ie: --no-{ac
         else:
             # capitalize every word in between '-'
             branch_list = self.branch.split('-')
             branch_list = [elem.capitalize() for elem in branch_list]
             return '-'.join(branch_list)
 
     def _query_props_set_by_mach(self, console_output=True, error_level=FATAL):
         mach_properties_path = os.path.join(
-            self.query_abs_dirs()['abs_obj_dir'], 'mach_build_properties.json'
+            self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json'
         )
         self.info("setting properties set by mach build. Looking in path: %s"
                   % mach_properties_path)
         if os.path.exists(mach_properties_path):
             with self.opened(mach_properties_path, error_level=error_level) as (fh, err):
                 build_props = json.load(fh)
                 if err:
                     self.log("%s exists but there was an error reading the "
@@ -1257,19 +1257,17 @@ or run without that action (ie: --no-{ac
                              error_level)
                 if console_output:
                     self.info("Properties set from 'mach build'")
                     self.info(pprint.pformat(build_props))
             for key, prop in build_props.iteritems():
                 if prop != 'UNKNOWN':
                     self.set_buildbot_property(key, prop, write_to_file=True)
         else:
-            self.log("Could not determine path for build properties. "
-                     "Does this exist: `%s` ?" % mach_properties_path,
-                     level=error_level)
+            self.info("No mach_build_properties.json found - not importing properties.")
 
     def generate_build_props(self, console_output=True, halt_on_failure=False):
         """sets props found from mach build and, in addition, buildid,
         sourcestamp,  appVersion, and appName."""
 
         error_level = ERROR
         if halt_on_failure:
             error_level = FATAL
@@ -1639,17 +1637,17 @@ or run without that action (ie: --no-{ac
             # the old package should live in source dir so we don't need to do
             # this for nighties since we clobber the whole work_dir in
             # clobber()
             self._rm_old_package()
         self._get_mozconfig()
         self._run_tooltool()
         self._create_mozbuild_dir()
         mach_props = os.path.join(
-            self.query_abs_dirs()['abs_obj_dir'], 'mach_build_properties.json'
+            self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json'
         )
         if os.path.exists(mach_props):
             self.info("Removing previous mach property file: %s" % mach_props)
             self.rmtree(mach_props)
 
     def build(self):
         """builds application."""
         env = self.query_build_env()
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -204,17 +204,17 @@ checksum:
 	@cat $(CHECKSUM_FILE)
 	@echo 'CHECKSUM FILE END'
 	$(SIGN_CHECKSUM_CMD)
 
 
 upload: checksum
 	$(PYTHON) $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) \
 		--package $(PACKAGE) \
-		--properties-file $(DIST)/upload-properties.json \
+		--properties-file $(DIST)/mach_build_properties.json \
 		$(UPLOAD_FILES) \
 		$(CHECKSUM_FILES)
 
 # source-package creates a source tarball from the files in MOZ_PKG_SRCDIR,
 # which is either set to a clean checkout or defaults to $topsrcdir
 source-package:
 	@echo 'Packaging source tarball...'
 	$(MKDIR) -p $(DIST)/$(PKG_SRCPACK_PATH)