Bug 978211 - add an automation/build target for post-build steps; r=glandium
authorMike Shal <mshal@mozilla.com>
Fri, 16 May 2014 14:37:31 -0400
changeset 209462 c0efa7f1e54c
parent 209461 038f7887f26d
child 209463 e92103cfb789
push id3857
push userraliiev@mozilla.com
push dateTue, 02 Sep 2014 16:39:23 +0000
treeherdermozilla-beta@5638b907b505 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs978211
milestone33.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 978211 - add an automation/build target for post-build steps; r=glandium
Makefile.in
build/gen_mach_buildprops.py
build/moz-automation.mk
client.mk
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/mach_commands.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -127,16 +127,17 @@ endif
 $(addprefix install-dist-,$(install_manifests)): install-dist-%: $(install_manifest_depends)
 	$(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$(DIST)/$* _build_manifests/install/dist_$*)
 
 .PHONY: install-tests
 install-manifests: install-tests
 install-tests: $(install_manifest_depends)
 	$(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )_tests _build_manifests/install/tests)
 
+include $(topsrcdir)/build/moz-automation.mk
 
 # _tests should be purged during cleaning. However, we don't want it purged
 # during PGO builds because it contains some auto-generated files.
 ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),)
 GARBAGE_DIRS += _tests
 endif
 
 # Windows PGO builds don't perform a clean before the 2nd pass. So, we want
@@ -163,17 +164,17 @@ export:: install-dist-sdk
 ifndef JS_STANDALONE
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
 include $(topsrcdir)/testing/testsuite-targets.mk
 endif
 endif
 
 default all::
-	$(call BUILDSTATUS,TIERS export $(if $(COMPILE_ENVIRONMENT),$(if $(MOZ_PSEUDO_DERECURSE),compile ))libs tools)
+	$(call BUILDSTATUS,TIERS export $(if $(COMPILE_ENVIRONMENT),$(if $(MOZ_PSEUDO_DERECURSE),compile ))libs tools $(if $(MOZ_AUTOMATION),$(MOZ_AUTOMATION_TIERS)))
 
 include $(topsrcdir)/config/rules.mk
 
 distclean::
 	$(RM) $(DIST_GARBAGE)
 
 ifeq ($(OS_ARCH),WINNT)
 # we want to copy PDB files on Windows
@@ -246,16 +247,19 @@ endif
 
 # MOZ_SOURCE_STAMP is defined in package-name.mk with a deferred assignment.
 # exporting it makes make run its $(shell) command for each invoked submake,
 # so transform it to an immediate assignment.
 MOZ_SOURCE_STAMP := $(MOZ_SOURCE_STAMP)
 export MOZ_SOURCE_STAMP
 endif
 
+update-packaging:
+	$(MAKE) -C tools/update-packaging
+
 #XXX: this is a hack, since we don't want to clobber for MSVC
 # PGO support, but we can't do this test in client.mk
 ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 # No point in clobbering if PGO has been explicitly disabled.
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 maybe_clobber_profiledbuild: clean
 else
 maybe_clobber_profiledbuild:
new file mode 100644
--- /dev/null
+++ b/build/gen_mach_buildprops.py
@@ -0,0 +1,79 @@
+#!/usr/bin/python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys, os, sha, json, re
+from argparse import ArgumentParser
+
+def getFileHashAndSize(filename):
+    sha1Hash = 'UNKNOWN'
+    size = 'UNKNOWN'
+
+    try:
+        # open in binary mode to make sure we get consistent results
+        # across all platforms
+        f = open(filename, "rb")
+        shaObj = sha.new(f.read())
+        sha1Hash = shaObj.hexdigest()
+
+        size = os.path.getsize(filename)
+    except:
+        pass
+
+    return (sha1Hash, size)
+
+def getMarProperties(filename):
+    (complete_mar_hash, complete_mar_size) = getFileHashAndSize(filename)
+    return {
+        'completeMarFilename': os.path.basename(filename),
+        'completeMarSize': complete_mar_size,
+        'completeMarHash': complete_mar_hash,
+    }
+
+def getUrlProperties(filename):
+    # let's create a switch case using name-spaces/dict
+    # rather than a long if/else with duplicate code
+    property_conditions = [
+        # key: property name, value: condition
+        ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
+                       m.endswith('crashreporter-symbols-full.zip')),
+        ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
+        ('unsignedApkUrl', lambda m: m.endswith('apk') and
+                           'unsigned-unaligned' in m),
+        ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
+        ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
+        ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
+        # packageUrl must be last!
+        ('packageUrl', lambda m: True),
+    ]
+    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
+    properties = {}
+
+    with open(filename) as f:
+        for line in f:
+            m = url_re.match(line)
+            if m:
+                m = m.group(1)
+                for prop, condition in property_conditions:
+                    if condition(m):
+                        properties.update({prop: m})
+                        break
+    return properties
+
+if __name__ == '__main__':
+    parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
+    parser.add_argument("--complete-mar-file", required=True,
+                        action="store", dest="complete_mar_file",
+                        help="Path to the complete MAR file, relative to the objdir.")
+    parser.add_argument("--upload-output", required=True,
+                        action="store", dest="upload_output",
+                        help="Path to the text output of 'make upload'")
+    args = parser.parse_args()
+
+    json_data = getMarProperties(args.complete_mar_file)
+    json_data.update(getUrlProperties(args.upload_output))
+
+    with open('mach_build_properties.json', 'w') as outfile:
+        json.dump(json_data, outfile, indent=4)
new file mode 100644
--- /dev/null
+++ b/build/moz-automation.mk
@@ -0,0 +1,66 @@
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
+
+# Log file from the 'make upload' step. We need this to parse out the URLs of
+# the uploaded files.
+AUTOMATION_UPLOAD_OUTPUT = $(DIST)/automation-upload.txt
+
+# Automation build steps. Everything in MOZ_AUTOMATION_TIERS also gets used in
+# TIERS for mach display. As such, the MOZ_AUTOMATION_TIERS are roughly sorted
+# here in the order that they will be executed (since mach doesn't know of the
+# dependencies between them).
+
+MOZ_AUTOMATION_TIERS += package-tests
+MOZ_AUTOMATION_TIERS += buildsymbols
+
+ifdef NIGHTLY_BUILD
+MOZ_AUTOMATION_TIERS += uploadsymbols
+endif
+
+ifdef MOZ_UPDATE_PACKAGING
+MOZ_AUTOMATION_TIERS += update-packaging
+automation/upload: automation/update-packaging
+automation/update-packaging: automation/package
+endif
+
+MOZ_AUTOMATION_TIERS += package
+MOZ_AUTOMATION_TIERS += check
+
+ifndef MOZ_ASAN
+MOZ_AUTOMATION_TIERS += l10n-check
+automation/l10n-check: automation/package
+endif
+
+MOZ_AUTOMATION_TIERS += upload
+
+automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
+	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
+
+# Dependencies between automation build steps
+automation/uploadsymbols: automation/buildsymbols
+
+automation/upload: automation/package
+automation/upload: automation/package-tests
+automation/upload: automation/buildsymbols
+
+# automation/package and automation/check should depend on build (which is
+# implicit due to the way client.mk invokes automation/build), but buildsymbols
+# changes the binaries/libs, and that's what we package/test.
+automation/package: automation/buildsymbols
+automation/check: automation/buildsymbols
+
+# make check runs with the keep-going flag so we can see all the failures
+AUTOMATION_EXTRA_CMDLINE-check = -k
+
+# We need the log from make upload to grep it for urls in order to set
+# properties.
+AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT)
+
+automation/%:
+	$(call BUILDSTATUS,TIER_START $*)
+	@$(MAKE) $* $(AUTOMATION_EXTRA_CMDLINE-$*)
+	$(call BUILDSTATUS,TIER_FINISH $*)
--- a/client.mk
+++ b/client.mk
@@ -171,17 +171,17 @@ MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C 
 
 endif # MOZ_BUILD_PROJECTS
 
 # 'configure' scripts generated by autoconf.
 CONFIGURES := $(TOPSRCDIR)/configure
 CONFIGURES += $(TOPSRCDIR)/js/src/configure
 
 # Make targets that are going to be passed to the real build system
-OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check
+OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check automation/build
 
 #######################################################################
 # Rules
 
 # The default rule is build
 build::
 	$(MAKE) -f $(TOPSRCDIR)/client.mk $(if $(MOZ_PGO),profiledbuild,realbuild)
 
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -60,17 +60,17 @@ class TierStatus(object):
 
     The build system is organized into linear phases called tiers. Each tier
     executes in the order it was defined, 1 at a time.
     """
 
     def __init__(self, resources):
         """Accepts a SystemResourceMonitor to record results against."""
         self.tiers = OrderedDict()
-        self.active_tier = None
+        self.active_tiers = set()
         self.resources = resources
 
     def set_tiers(self, tiers):
         """Record the set of known tiers."""
         for tier in tiers:
             self.tiers[tier] = dict(
                 begin_time=None,
                 finish_time=None,
@@ -79,27 +79,28 @@ class TierStatus(object):
 
     def begin_tier(self, tier):
         """Record that execution of a tier has begun."""
         t = self.tiers[tier]
         # We should ideally use a monotonic clock here. Unfortunately, we won't
         # have one until Python 3.
         t['begin_time'] = time.time()
         self.resources.begin_phase(tier)
-        self.active_tier = tier
+        self.active_tiers.add(tier)
 
     def finish_tier(self, tier):
         """Record that execution of a tier has finished."""
         t = self.tiers[tier]
         t['finish_time'] = time.time()
         t['duration'] = self.resources.finish_phase(tier)
+        self.active_tiers.remove(tier)
 
     def tier_status(self):
         for tier, state in self.tiers.items():
-            active = self.active_tier == tier
+            active = tier in self.active_tiers
             finished = state['finish_time'] is not None
 
             yield tier, active, finished
 
     def tiered_resource_usage(self):
         """Obtains an object containing resource usage for tiers.
 
         The returned object is suitable for serialization.
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -362,16 +362,28 @@ class Build(MachCommandBase):
                         break
             else:
                 monitor.start_resource_recording()
                 status = self._run_make(srcdir=True, filename='client.mk',
                     line_handler=output.on_line, log=False, print_directory=False,
                     allow_parallel=False, ensure_exit_code=False, num_jobs=jobs,
                     silent=not verbose, force_pymake=pymake)
 
+                make_extra = self.mozconfig['make_extra'] or []
+                make_extra = dict(m.split('=', 1) for m in make_extra)
+
+                moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get('export MOZ_AUTOMATION', None)
+                if moz_automation and status == 0:
+                    # Default to 1 job until we have make-4.0 in for output
+                    # buffering.
+                    status = self._run_make(target='automation/build',
+                        line_handler=output.on_line, log=False, print_directory=False,
+                        allow_parallel=False, ensure_exit_code=False, num_jobs=1,
+                        silent=not verbose, force_pymake=pymake)
+
                 self.log(logging.WARNING, 'warning_summary',
                     {'count': len(monitor.warnings_database)},
                     '{count} compiler warnings present.')
 
             monitor.finish(record_usage=status==0)
 
         high_finder, finder_percent = monitor.have_high_finder_usage()
         if high_finder: