author | Mike Shal <mshal@mozilla.com> |
Fri, 16 May 2014 14:37:31 -0400 | |
changeset 210584 | c0efa7f1e54c2dfbc9dfa77c49bf0411bc8b891d |
parent 210583 | 038f7887f26d9e2bfc7f95129b994174c5eab380 |
child 210585 | e92103cfb78948e7596d0d6843831e320f9354ea |
push id | 515 |
push user | raliiev@mozilla.com |
push date | Mon, 06 Oct 2014 12:51:51 +0000 |
treeherder | mozilla-release@267c7a481bef [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | glandium |
bugs | 978211 |
milestone | 33.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/Makefile.in +++ b/Makefile.in @@ -127,16 +127,17 @@ endif $(addprefix install-dist-,$(install_manifests)): install-dist-%: $(install_manifest_depends) $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$(DIST)/$* _build_manifests/install/dist_$*) .PHONY: install-tests install-manifests: install-tests install-tests: $(install_manifest_depends) $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )_tests _build_manifests/install/tests) +include $(topsrcdir)/build/moz-automation.mk # _tests should be purged during cleaning. However, we don't want it purged # during PGO builds because it contains some auto-generated files. ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),) GARBAGE_DIRS += _tests endif # Windows PGO builds don't perform a clean before the 2nd pass. So, we want @@ -163,17 +164,17 @@ export:: install-dist-sdk ifndef JS_STANDALONE ifdef ENABLE_TESTS # Additional makefile targets to call automated test suites include $(topsrcdir)/testing/testsuite-targets.mk endif endif default all:: - $(call BUILDSTATUS,TIERS export $(if $(COMPILE_ENVIRONMENT),$(if $(MOZ_PSEUDO_DERECURSE),compile ))libs tools) + $(call BUILDSTATUS,TIERS export $(if $(COMPILE_ENVIRONMENT),$(if $(MOZ_PSEUDO_DERECURSE),compile ))libs tools $(if $(MOZ_AUTOMATION),$(MOZ_AUTOMATION_TIERS))) include $(topsrcdir)/config/rules.mk distclean:: $(RM) $(DIST_GARBAGE) ifeq ($(OS_ARCH),WINNT) # we want to copy PDB files on Windows @@ -246,16 +247,19 @@ endif # MOZ_SOURCE_STAMP is defined in package-name.mk with a deferred assignment. # exporting it makes make run its $(shell) command for each invoked submake, # so transform it to an immediate assignment. MOZ_SOURCE_STAMP := $(MOZ_SOURCE_STAMP) export MOZ_SOURCE_STAMP endif +update-packaging: + $(MAKE) -C tools/update-packaging + #XXX: this is a hack, since we don't want to clobber for MSVC # PGO support, but we can't do this test in client.mk ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_) # No point in clobbering if PGO has been explicitly disabled. ifndef NO_PROFILE_GUIDED_OPTIMIZE maybe_clobber_profiledbuild: clean else maybe_clobber_profiledbuild:
new file mode 100644 --- /dev/null +++ b/build/gen_mach_buildprops.py @@ -0,0 +1,79 @@ +#!/usr/bin/python +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys, os, sha, json, re +from argparse import ArgumentParser + +def getFileHashAndSize(filename): + sha1Hash = 'UNKNOWN' + size = 'UNKNOWN' + + try: + # open in binary mode to make sure we get consistent results + # across all platforms + f = open(filename, "rb") + shaObj = sha.new(f.read()) + sha1Hash = shaObj.hexdigest() + + size = os.path.getsize(filename) + except: + pass + + return (sha1Hash, size) + +def getMarProperties(filename): + (complete_mar_hash, complete_mar_size) = getFileHashAndSize(filename) + return { + 'completeMarFilename': os.path.basename(filename), + 'completeMarSize': complete_mar_size, + 'completeMarHash': complete_mar_hash, + } + +def getUrlProperties(filename): + # let's create a switch case using name-spaces/dict + # rather than a long if/else with duplicate code + property_conditions = [ + # key: property name, value: condition + ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or + m.endswith('crashreporter-symbols-full.zip')), + ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))), + ('unsignedApkUrl', lambda m: m.endswith('apk') and + 'unsigned-unaligned' in m), + ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m), + ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')), + ('completeMarUrl', lambda m: m.endswith('.complete.mar')), + # packageUrl must be last! + ('packageUrl', lambda m: True), + ] + url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''') + properties = {} + + with open(filename) as f: + for line in f: + m = url_re.match(line) + if m: + m = m.group(1) + for prop, condition in property_conditions: + if condition(m): + properties.update({prop: m}) + break + return properties + +if __name__ == '__main__': + parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.') + parser.add_argument("--complete-mar-file", required=True, + action="store", dest="complete_mar_file", + help="Path to the complete MAR file, relative to the objdir.") + parser.add_argument("--upload-output", required=True, + action="store", dest="upload_output", + help="Path to the text output of 'make upload'") + args = parser.parse_args() + + json_data = getMarProperties(args.complete_mar_file) + json_data.update(getUrlProperties(args.upload_output)) + + with open('mach_build_properties.json', 'w') as outfile: + json.dump(json_data, outfile, indent=4)
new file mode 100644 --- /dev/null +++ b/build/moz-automation.mk @@ -0,0 +1,66 @@ +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk + +# Log file from the 'make upload' step. We need this to parse out the URLs of +# the uploaded files. +AUTOMATION_UPLOAD_OUTPUT = $(DIST)/automation-upload.txt + +# Automation build steps. Everything in MOZ_AUTOMATION_TIERS also gets used in +# TIERS for mach display. As such, the MOZ_AUTOMATION_TIERS are roughly sorted +# here in the order that they will be executed (since mach doesn't know of the +# dependencies between them). + +MOZ_AUTOMATION_TIERS += package-tests +MOZ_AUTOMATION_TIERS += buildsymbols + +ifdef NIGHTLY_BUILD +MOZ_AUTOMATION_TIERS += uploadsymbols +endif + +ifdef MOZ_UPDATE_PACKAGING +MOZ_AUTOMATION_TIERS += update-packaging +automation/upload: automation/update-packaging +automation/update-packaging: automation/package +endif + +MOZ_AUTOMATION_TIERS += package +MOZ_AUTOMATION_TIERS += check + +ifndef MOZ_ASAN +MOZ_AUTOMATION_TIERS += l10n-check +automation/l10n-check: automation/package +endif + +MOZ_AUTOMATION_TIERS += upload + +automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS)) + $(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT) + +# Dependencies between automation build steps +automation/uploadsymbols: automation/buildsymbols + +automation/upload: automation/package +automation/upload: automation/package-tests +automation/upload: automation/buildsymbols + +# automation/package and automation/check should depend on build (which is +# implicit due to the way client.mk invokes automation/build), but buildsymbols +# changes the binaries/libs, and that's what we package/test. +automation/package: automation/buildsymbols +automation/check: automation/buildsymbols + +# make check runs with the keep-going flag so we can see all the failures +AUTOMATION_EXTRA_CMDLINE-check = -k + +# We need the log from make upload to grep it for urls in order to set +# properties. +AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT) + +automation/%: + $(call BUILDSTATUS,TIER_START $*) + @$(MAKE) $* $(AUTOMATION_EXTRA_CMDLINE-$*) + $(call BUILDSTATUS,TIER_FINISH $*)
--- a/client.mk +++ b/client.mk @@ -171,17 +171,17 @@ MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C endif # MOZ_BUILD_PROJECTS # 'configure' scripts generated by autoconf. CONFIGURES := $(TOPSRCDIR)/configure CONFIGURES += $(TOPSRCDIR)/js/src/configure # Make targets that are going to be passed to the real build system -OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check +OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check automation/build ####################################################################### # Rules # The default rule is build build:: $(MAKE) -f $(TOPSRCDIR)/client.mk $(if $(MOZ_PGO),profiledbuild,realbuild)
--- a/python/mozbuild/mozbuild/controller/building.py +++ b/python/mozbuild/mozbuild/controller/building.py @@ -60,17 +60,17 @@ class TierStatus(object): The build system is organized into linear phases called tiers. Each tier executes in the order it was defined, 1 at a time. """ def __init__(self, resources): """Accepts a SystemResourceMonitor to record results against.""" self.tiers = OrderedDict() - self.active_tier = None + self.active_tiers = set() self.resources = resources def set_tiers(self, tiers): """Record the set of known tiers.""" for tier in tiers: self.tiers[tier] = dict( begin_time=None, finish_time=None, @@ -79,27 +79,28 @@ class TierStatus(object): def begin_tier(self, tier): """Record that execution of a tier has begun.""" t = self.tiers[tier] # We should ideally use a monotonic clock here. Unfortunately, we won't # have one until Python 3. t['begin_time'] = time.time() self.resources.begin_phase(tier) - self.active_tier = tier + self.active_tiers.add(tier) def finish_tier(self, tier): """Record that execution of a tier has finished.""" t = self.tiers[tier] t['finish_time'] = time.time() t['duration'] = self.resources.finish_phase(tier) + self.active_tiers.remove(tier) def tier_status(self): for tier, state in self.tiers.items(): - active = self.active_tier == tier + active = tier in self.active_tiers finished = state['finish_time'] is not None yield tier, active, finished def tiered_resource_usage(self): """Obtains an object containing resource usage for tiers. The returned object is suitable for serialization.
--- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -362,16 +362,28 @@ class Build(MachCommandBase): break else: monitor.start_resource_recording() status = self._run_make(srcdir=True, filename='client.mk', line_handler=output.on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, force_pymake=pymake) + make_extra = self.mozconfig['make_extra'] or [] + make_extra = dict(m.split('=', 1) for m in make_extra) + + moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get('export MOZ_AUTOMATION', None) + if moz_automation and status == 0: + # Default to 1 job until we have make-4.0 in for output + # buffering. + status = self._run_make(target='automation/build', + line_handler=output.on_line, log=False, print_directory=False, + allow_parallel=False, ensure_exit_code=False, num_jobs=1, + silent=not verbose, force_pymake=pymake) + self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') monitor.finish(record_usage=status==0) high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: