Bug 1087104 - Implement partial mar generation in make for 'mach build'; r=glandium a=NPOTB
authorMike Shal <mshal@mozilla.com>
Fri, 24 Oct 2014 18:24:33 -0400
changeset 225994 ee2c3cfb4a7b
parent 225993 72d8ba95b2db
child 225995 dc18ad2b4816
push id4099
push usermshal@mozilla.com
push date2014-11-07 15:57 +0000
treeherdermozilla-beta@dc18ad2b4816 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium, NPOTB
bugs1087104
milestone34.0
Bug 1087104 - Implement partial mar generation in make for 'mach build'; r=glandium a=NPOTB
build/gen_mach_buildprops.py
build/moz-automation.mk
tools/update-packaging/Makefile.in
--- a/build/gen_mach_buildprops.py
+++ b/build/gen_mach_buildprops.py
@@ -24,39 +24,41 @@ def getFileHashAndSize(filename):
         sha512Hash = shaObj.hexdigest()
 
         size = os.path.getsize(filename)
     except:
         pass
 
     return (sha512Hash, size)
 
-def getMarProperties(filename):
+def getMarProperties(filename, partial=False):
     if not os.path.exists(filename):
         return {}
-    (complete_mar_hash, complete_mar_size) = getFileHashAndSize(filename)
+    (mar_hash, mar_size) = getFileHashAndSize(filename)
+    martype = 'partial' if partial else 'complete'
     return {
-        'completeMarFilename': os.path.basename(filename),
-        'completeMarSize': complete_mar_size,
-        'completeMarHash': complete_mar_hash,
+        '%sMarFilename' % martype: os.path.basename(filename),
+        '%sMarSize' % martype: mar_size,
+        '%sMarHash' % martype: mar_hash,
     }
 
 def getUrlProperties(filename):
     # let's create a switch case using name-spaces/dict
     # rather than a long if/else with duplicate code
     property_conditions = [
         # key: property name, value: condition
         ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
                        m.endswith('crashreporter-symbols-full.zip')),
         ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
         ('unsignedApkUrl', lambda m: m.endswith('apk') and
                            'unsigned-unaligned' in m),
         ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
         ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
         ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
+        ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
         # packageUrl must be last!
         ('packageUrl', lambda m: True),
     ]
     url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
     properties = {}
 
     try:
         with open(filename) as f:
@@ -74,18 +76,23 @@ def getUrlProperties(filename):
         properties = {prop: 'UNKNOWN' for prop, condition in property_conditions}
     return properties
 
 if __name__ == '__main__':
     parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
     parser.add_argument("--complete-mar-file", required=True,
                         action="store", dest="complete_mar_file",
                         help="Path to the complete MAR file, relative to the objdir.")
+    parser.add_argument("--partial-mar-file", required=False,
+                        action="store", dest="partial_mar_file",
+                        help="Path to the partial MAR file, relative to the objdir.")
     parser.add_argument("--upload-output", required=True,
                         action="store", dest="upload_output",
                         help="Path to the text output of 'make upload'")
     args = parser.parse_args()
 
     json_data = getMarProperties(args.complete_mar_file)
+    if args.partial_mar_file:
+        json_data.update(getMarProperties(args.partial_mar_file, partial=True))
     json_data.update(getUrlProperties(args.upload_output))
 
     with open('mach_build_properties.json', 'w') as outfile:
         json.dump(json_data, outfile, indent=4)
--- a/build/moz-automation.mk
+++ b/build/moz-automation.mk
@@ -81,17 +81,17 @@ automation/pretty-package: automation/bu
 # conflicts in writing to the same files.
 automation/installer: automation/pretty-installer
 automation/package: automation/pretty-package
 automation/package-tests: automation/pretty-package-tests
 automation/l10n-check: automation/pretty-l10n-check
 automation/update-packaging: automation/pretty-update-packaging
 
 automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
-	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
+	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) $(addprefix --partial-mar-file ,$(wildcard $(DIST)/$(PARTIAL_MAR))) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
 
 # We need the log from make upload to grep it for urls in order to set
 # properties.
 AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT)
 
 # Note: We have to force -j1 here, at least until bug 1036563 is fixed.
 AUTOMATION_EXTRA_CMDLINE-l10n-check = -j1
 AUTOMATION_EXTRA_CMDLINE-pretty-l10n-check = -j1
--- a/tools/update-packaging/Makefile.in
+++ b/tools/update-packaging/Makefile.in
@@ -26,17 +26,17 @@ endif
 else
 PACKAGE_DIR	= $(PACKAGE_BASE_DIR)/$(MOZ_PKG_DIR)
 endif
 
 MAR_BIN	= $(LIBXUL_DIST)/host/bin/mar$(HOST_BIN_SUFFIX)
 MBSDIFF_BIN	= $(LIBXUL_DIST)/host/bin/mbsdiff$(HOST_BIN_SUFFIX)
 
 OVERRIDE_DEFAULT_GOAL := full-update
-full-update:: complete-patch
+full-update:: complete-patch $(if $(MOZ_AUTOMATION_UPDATE_PACKAGING),automation-partial-patch)
 
 ifeq ($(OS_TARGET), WINNT)
 MOZ_PKG_FORMAT	:= SFX7Z
 UNPACKAGE	= '$(subst $(DIST),$(_ABS_DIST),$(INSTALLER_PACKAGE))'
 ifdef AB_CD
 UNPACKAGE	= '$(PACKAGE_BASE_DIR)/$(PACKAGE)'
 endif
 endif
@@ -56,24 +56,46 @@ dir-stage := $(call mkdir_deps,$(STAGE_D
 complete-patch:: $(dir-stage)
 ifeq ($(OS_TARGET), WINNT)
 	test -f $(UNPACKAGE)
 	$(RM) -rf '$(PACKAGE_DIR)'
 	cd $(PACKAGE_BASE_DIR) && $(INNER_UNMAKE_PACKAGE)
 endif
 	MAR=$(MAR_BIN) \
 	  $(srcdir)/make_full_update.sh \
-	  '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).complete.mar' \
+	  '$(DIST)/$(COMPLETE_MAR)' \
 	  '$(PACKAGE_DIR)'
 ifdef MOZ_SIGN_CMD
-	$(MOZ_SIGN_CMD) -f mar '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).complete.mar'
+	$(MOZ_SIGN_CMD) -f mar '$(DIST)/$(COMPLETE_MAR)'
 endif
 
 partial-patch:: $(dir-stage)
 	MAR=$(MAR_BIN) \
 	MBSDIFF=$(MBSDIFF_BIN) \
 	  $(srcdir)/make_incremental_update.sh \
 	  '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar' \
 	  '$(SRC_BUILD)' \
 	  '$(DST_BUILD)'
 ifdef MOZ_SIGN_CMD
 	$(MOZ_SIGN_CMD) -f mar '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar'
 endif
+
+automation-partial-patch: complete-patch
+	rm -rf current current.work previous
+	mkdir current previous
+	latestmar=$$(ssh -l $(UPLOAD_USER) -i $(UPLOAD_SSH_KEY) $(UPLOAD_HOST) 'ls -1t $(LATEST_MAR_DIR) | grep $(MOZ_PKG_PLATFORM).complete.mar$$ | head -n 1'); \
+	  if test -n "$$latestmar"; then \
+	    wget -O $(STAGE_DIR)/previous.mar http://$(UPLOAD_HOST)/$(LATEST_MAR_DIR)/$$latestmar && \
+	    (cd previous; \
+	      MAR=$(MAR_BIN) perl $(topsrcdir)/tools/update-packaging/unwrap_full_update.pl '../$(STAGE_DIR)/previous.mar') && \
+	    (cd current; \
+	      MAR=$(MAR_BIN) perl $(topsrcdir)/tools/update-packaging/unwrap_full_update.pl '../$(DIST)/$(COMPLETE_MAR)') && \
+	    find current -name \*.pgc -print -delete && \
+	    find previous -name \*.pgc -print -delete && \
+	    rm -f $(STAGE_DIR)/*.partial.*.mar && \
+	    SRC_BUILD_ID=$$(python $(topsrcdir)/config/printconfigsetting.py $$(find previous -maxdepth 4 -type f -name application.ini) App BuildID) \
+	    DST_BUILD_ID=$$(cat $(DEPTH)/config/buildid) \
+	    SRC_BUILD=previous DST_BUILD=current \
+	      $(MAKE) partial-patch && \
+	    rm -f $(STAGE_DIR)/previous.mar; \
+	  else \
+	    echo "No previous MAR found; not creating a partial MAR"; \
+	  fi