Bug 1087104 - Implement partial mar generation in make for 'mach build'; r=glandium
authorMike Shal <mshal@mozilla.com>
Fri, 24 Oct 2014 18:24:33 -0400
changeset 212319 44aadcfb5f5788e9a815e8fc2b289c1326c3cdb2
parent 212318 f8e9b337e5d7ad0aa6d6269d794d9ec1f4ab2df4
child 212320 32e40c42dc81586848b158294d0358c9fb2ec657
push id50935
push usermshal@mozilla.com
push dateFri, 24 Oct 2014 22:36:21 +0000
treeherdermozilla-inbound@44aadcfb5f57 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1087104
milestone36.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1087104 - Implement partial mar generation in make for 'mach build'; r=glandium
build/gen_mach_buildprops.py
build/moz-automation.mk
tools/update-packaging/Makefile.in
--- a/build/gen_mach_buildprops.py
+++ b/build/gen_mach_buildprops.py
@@ -24,39 +24,41 @@ def getFileHashAndSize(filename):
         sha512Hash = shaObj.hexdigest()
 
         size = os.path.getsize(filename)
     except:
         pass
 
     return (sha512Hash, size)
 
-def getMarProperties(filename):
+def getMarProperties(filename, partial=False):
     if not os.path.exists(filename):
         return {}
-    (complete_mar_hash, complete_mar_size) = getFileHashAndSize(filename)
+    (mar_hash, mar_size) = getFileHashAndSize(filename)
+    martype = 'partial' if partial else 'complete'
     return {
-        'completeMarFilename': os.path.basename(filename),
-        'completeMarSize': complete_mar_size,
-        'completeMarHash': complete_mar_hash,
+        '%sMarFilename' % martype: os.path.basename(filename),
+        '%sMarSize' % martype: mar_size,
+        '%sMarHash' % martype: mar_hash,
     }
 
 def getUrlProperties(filename):
     # let's create a switch case using name-spaces/dict
     # rather than a long if/else with duplicate code
     property_conditions = [
         # key: property name, value: condition
         ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
                        m.endswith('crashreporter-symbols-full.zip')),
         ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
         ('unsignedApkUrl', lambda m: m.endswith('apk') and
                            'unsigned-unaligned' in m),
         ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
         ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
         ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
+        ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
         # packageUrl must be last!
         ('packageUrl', lambda m: True),
     ]
     url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
     properties = {}
 
     try:
         with open(filename) as f:
@@ -74,18 +76,23 @@ def getUrlProperties(filename):
         properties = {prop: 'UNKNOWN' for prop, condition in property_conditions}
     return properties
 
 if __name__ == '__main__':
     parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
     parser.add_argument("--complete-mar-file", required=True,
                         action="store", dest="complete_mar_file",
                         help="Path to the complete MAR file, relative to the objdir.")
+    parser.add_argument("--partial-mar-file", required=False,
+                        action="store", dest="partial_mar_file",
+                        help="Path to the partial MAR file, relative to the objdir.")
     parser.add_argument("--upload-output", required=True,
                         action="store", dest="upload_output",
                         help="Path to the text output of 'make upload'")
     args = parser.parse_args()
 
     json_data = getMarProperties(args.complete_mar_file)
+    if args.partial_mar_file:
+        json_data.update(getMarProperties(args.partial_mar_file, partial=True))
     json_data.update(getUrlProperties(args.upload_output))
 
     with open('mach_build_properties.json', 'w') as outfile:
         json.dump(json_data, outfile, indent=4)
--- a/build/moz-automation.mk
+++ b/build/moz-automation.mk
@@ -81,17 +81,17 @@ automation/pretty-package: automation/bu
 # conflicts in writing to the same files.
 automation/installer: automation/pretty-installer
 automation/package: automation/pretty-package
 automation/package-tests: automation/pretty-package-tests
 automation/l10n-check: automation/pretty-l10n-check
 automation/update-packaging: automation/pretty-update-packaging
 
 automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
-	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
+	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) $(addprefix --partial-mar-file ,$(wildcard $(DIST)/$(PARTIAL_MAR))) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
 
 # We need the log from make upload to grep it for urls in order to set
 # properties.
 AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT)
 
 # Note: We have to force -j1 here, at least until bug 1036563 is fixed.
 AUTOMATION_EXTRA_CMDLINE-l10n-check = -j1
 AUTOMATION_EXTRA_CMDLINE-pretty-l10n-check = -j1
--- a/tools/update-packaging/Makefile.in
+++ b/tools/update-packaging/Makefile.in
@@ -26,17 +26,17 @@ endif
 else
 PACKAGE_DIR	= $(PACKAGE_BASE_DIR)/$(MOZ_PKG_DIR)
 endif
 
 MAR_BIN	= $(LIBXUL_DIST)/host/bin/mar$(HOST_BIN_SUFFIX)
 MBSDIFF_BIN	= $(LIBXUL_DIST)/host/bin/mbsdiff$(HOST_BIN_SUFFIX)
 
 OVERRIDE_DEFAULT_GOAL := full-update
-full-update:: complete-patch
+full-update:: complete-patch $(if $(MOZ_AUTOMATION_UPDATE_PACKAGING),automation-partial-patch)
 
 ifeq ($(OS_TARGET), WINNT)
 MOZ_PKG_FORMAT	:= SFX7Z
 UNPACKAGE	= '$(subst $(DIST),$(_ABS_DIST),$(INSTALLER_PACKAGE))'
 ifdef AB_CD
 UNPACKAGE	= '$(PACKAGE_BASE_DIR)/$(PACKAGE)'
 endif
 endif
@@ -56,24 +56,46 @@ dir-stage := $(call mkdir_deps,$(STAGE_D
 complete-patch:: $(dir-stage)
 ifeq ($(OS_TARGET), WINNT)
 	test -f $(UNPACKAGE)
 	$(RM) -rf '$(PACKAGE_DIR)'
 	cd $(PACKAGE_BASE_DIR) && $(INNER_UNMAKE_PACKAGE)
 endif
 	MAR=$(MAR_BIN) \
 	  $(srcdir)/make_full_update.sh \
-	  '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).complete.mar' \
+	  '$(DIST)/$(COMPLETE_MAR)' \
 	  '$(PACKAGE_DIR)'
 ifdef MOZ_SIGN_CMD
-	$(MOZ_SIGN_CMD) -f mar '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).complete.mar'
+	$(MOZ_SIGN_CMD) -f mar '$(DIST)/$(COMPLETE_MAR)'
 endif
 
 partial-patch:: $(dir-stage)
 	MAR=$(MAR_BIN) \
 	MBSDIFF=$(MBSDIFF_BIN) \
 	  $(srcdir)/make_incremental_update.sh \
 	  '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar' \
 	  '$(SRC_BUILD)' \
 	  '$(DST_BUILD)'
 ifdef MOZ_SIGN_CMD
 	$(MOZ_SIGN_CMD) -f mar '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar'
 endif
+
+automation-partial-patch: complete-patch
+	rm -rf current current.work previous
+	mkdir current previous
+	latestmar=$$(ssh -l $(UPLOAD_USER) -i $(UPLOAD_SSH_KEY) $(UPLOAD_HOST) 'ls -1t $(LATEST_MAR_DIR) | grep $(MOZ_PKG_PLATFORM).complete.mar$$ | head -n 1'); \
+	  if test -n "$$latestmar"; then \
+	    wget -O $(STAGE_DIR)/previous.mar http://$(UPLOAD_HOST)/$(LATEST_MAR_DIR)/$$latestmar && \
+	    (cd previous; \
+	      MAR=$(MAR_BIN) perl $(topsrcdir)/tools/update-packaging/unwrap_full_update.pl '../$(STAGE_DIR)/previous.mar') && \
+	    (cd current; \
+	      MAR=$(MAR_BIN) perl $(topsrcdir)/tools/update-packaging/unwrap_full_update.pl '../$(DIST)/$(COMPLETE_MAR)') && \
+	    find current -name \*.pgc -print -delete && \
+	    find previous -name \*.pgc -print -delete && \
+	    rm -f $(STAGE_DIR)/*.partial.*.mar && \
+	    SRC_BUILD_ID=$$(python $(topsrcdir)/config/printconfigsetting.py $$(find previous -maxdepth 4 -type f -name application.ini) App BuildID) \
+	    DST_BUILD_ID=$$(cat $(DEPTH)/config/buildid) \
+	    SRC_BUILD=previous DST_BUILD=current \
+	      $(MAKE) partial-patch && \
+	    rm -f $(STAGE_DIR)/previous.mar; \
+	  else \
+	    echo "No previous MAR found; not creating a partial MAR"; \
+	  fi