Bug 978211 - add an automation/build target for post-build steps; r=glandium
authorMike Shal <mshal@mozilla.com>
Fri, 16 May 2014 14:37:31 -0400
changeset 188697 c0efa7f1e54c2dfbc9dfa77c49bf0411bc8b891d
parent 188696 038f7887f26d9e2bfc7f95129b994174c5eab380
child 188698 e92103cfb78948e7596d0d6843831e320f9354ea
push id7289
push userkwierso@gmail.com
push dateSat, 14 Jun 2014 00:51:32 +0000
treeherderfx-team@161b482a0d55 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs978211
milestone33.0a1
Bug 978211 - add an automation/build target for post-build steps; r=glandium
Makefile.in
build/gen_mach_buildprops.py
build/moz-automation.mk
client.mk
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/mach_commands.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -127,16 +127,17 @@ endif
 $(addprefix install-dist-,$(install_manifests)): install-dist-%: $(install_manifest_depends)
 	$(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$(DIST)/$* _build_manifests/install/dist_$*)
 
 .PHONY: install-tests
 install-manifests: install-tests
 install-tests: $(install_manifest_depends)
 	$(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )_tests _build_manifests/install/tests)
 
+include $(topsrcdir)/build/moz-automation.mk
 
 # _tests should be purged during cleaning. However, we don't want it purged
 # during PGO builds because it contains some auto-generated files.
 ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),)
 GARBAGE_DIRS += _tests
 endif
 
 # Windows PGO builds don't perform a clean before the 2nd pass. So, we want
@@ -163,17 +164,17 @@ export:: install-dist-sdk
 ifndef JS_STANDALONE
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
 include $(topsrcdir)/testing/testsuite-targets.mk
 endif
 endif
 
 default all::
-	$(call BUILDSTATUS,TIERS export $(if $(COMPILE_ENVIRONMENT),$(if $(MOZ_PSEUDO_DERECURSE),compile ))libs tools)
+	$(call BUILDSTATUS,TIERS export $(if $(COMPILE_ENVIRONMENT),$(if $(MOZ_PSEUDO_DERECURSE),compile ))libs tools $(if $(MOZ_AUTOMATION),$(MOZ_AUTOMATION_TIERS)))
 
 include $(topsrcdir)/config/rules.mk
 
 distclean::
 	$(RM) $(DIST_GARBAGE)
 
 ifeq ($(OS_ARCH),WINNT)
 # we want to copy PDB files on Windows
@@ -246,16 +247,19 @@ endif
 
 # MOZ_SOURCE_STAMP is defined in package-name.mk with a deferred assignment.
 # exporting it makes make run its $(shell) command for each invoked submake,
 # so transform it to an immediate assignment.
 MOZ_SOURCE_STAMP := $(MOZ_SOURCE_STAMP)
 export MOZ_SOURCE_STAMP
 endif
 
+update-packaging:
+	$(MAKE) -C tools/update-packaging
+
 #XXX: this is a hack, since we don't want to clobber for MSVC
 # PGO support, but we can't do this test in client.mk
 ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 # No point in clobbering if PGO has been explicitly disabled.
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 maybe_clobber_profiledbuild: clean
 else
 maybe_clobber_profiledbuild:
new file mode 100644
--- /dev/null
+++ b/build/gen_mach_buildprops.py
@@ -0,0 +1,79 @@
+#!/usr/bin/python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys, os, sha, json, re
+from argparse import ArgumentParser
+
+def getFileHashAndSize(filename):
+    sha1Hash = 'UNKNOWN'
+    size = 'UNKNOWN'
+
+    try:
+        # open in binary mode to make sure we get consistent results
+        # across all platforms
+        f = open(filename, "rb")
+        shaObj = sha.new(f.read())
+        sha1Hash = shaObj.hexdigest()
+
+        size = os.path.getsize(filename)
+    except:
+        pass
+
+    return (sha1Hash, size)
+
+def getMarProperties(filename):
+    (complete_mar_hash, complete_mar_size) = getFileHashAndSize(filename)
+    return {
+        'completeMarFilename': os.path.basename(filename),
+        'completeMarSize': complete_mar_size,
+        'completeMarHash': complete_mar_hash,
+    }
+
+def getUrlProperties(filename):
+    # let's create a switch case using name-spaces/dict
+    # rather than a long if/else with duplicate code
+    property_conditions = [
+        # key: property name, value: condition
+        ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
+                       m.endswith('crashreporter-symbols-full.zip')),
+        ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
+        ('unsignedApkUrl', lambda m: m.endswith('apk') and
+                           'unsigned-unaligned' in m),
+        ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
+        ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
+        ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
+        # packageUrl must be last!
+        ('packageUrl', lambda m: True),
+    ]
+    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
+    properties = {}
+
+    with open(filename) as f:
+        for line in f:
+            m = url_re.match(line)
+            if m:
+                m = m.group(1)
+                for prop, condition in property_conditions:
+                    if condition(m):
+                        properties.update({prop: m})
+                        break
+    return properties
+
+if __name__ == '__main__':
+    parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
+    parser.add_argument("--complete-mar-file", required=True,
+                        action="store", dest="complete_mar_file",
+                        help="Path to the complete MAR file, relative to the objdir.")
+    parser.add_argument("--upload-output", required=True,
+                        action="store", dest="upload_output",
+                        help="Path to the text output of 'make upload'")
+    args = parser.parse_args()
+
+    json_data = getMarProperties(args.complete_mar_file)
+    json_data.update(getUrlProperties(args.upload_output))
+
+    with open('mach_build_properties.json', 'w') as outfile:
+        json.dump(json_data, outfile, indent=4)
new file mode 100644
--- /dev/null
+++ b/build/moz-automation.mk
@@ -0,0 +1,66 @@
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
+
+# Log file from the 'make upload' step. We need this to parse out the URLs of
+# the uploaded files.
+AUTOMATION_UPLOAD_OUTPUT = $(DIST)/automation-upload.txt
+
+# Automation build steps. Everything in MOZ_AUTOMATION_TIERS also gets used in
+# TIERS for mach display. As such, the MOZ_AUTOMATION_TIERS are roughly sorted
+# here in the order that they will be executed (since mach doesn't know of the
+# dependencies between them).
+
+MOZ_AUTOMATION_TIERS += package-tests
+MOZ_AUTOMATION_TIERS += buildsymbols
+
+ifdef NIGHTLY_BUILD
+MOZ_AUTOMATION_TIERS += uploadsymbols
+endif
+
+ifdef MOZ_UPDATE_PACKAGING
+MOZ_AUTOMATION_TIERS += update-packaging
+automation/upload: automation/update-packaging
+automation/update-packaging: automation/package
+endif
+
+MOZ_AUTOMATION_TIERS += package
+MOZ_AUTOMATION_TIERS += check
+
+ifndef MOZ_ASAN
+MOZ_AUTOMATION_TIERS += l10n-check
+automation/l10n-check: automation/package
+endif
+
+MOZ_AUTOMATION_TIERS += upload
+
+automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
+	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
+
+# Dependencies between automation build steps
+automation/uploadsymbols: automation/buildsymbols
+
+automation/upload: automation/package
+automation/upload: automation/package-tests
+automation/upload: automation/buildsymbols
+
+# automation/package and automation/check should depend on build (which is
+# implicit due to the way client.mk invokes automation/build), but buildsymbols
+# changes the binaries/libs, and that's what we package/test.
+automation/package: automation/buildsymbols
+automation/check: automation/buildsymbols
+
+# make check runs with the keep-going flag so we can see all the failures
+AUTOMATION_EXTRA_CMDLINE-check = -k
+
+# We need the log from make upload to grep it for urls in order to set
+# properties.
+AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT)
+
+automation/%:
+	$(call BUILDSTATUS,TIER_START $*)
+	@$(MAKE) $* $(AUTOMATION_EXTRA_CMDLINE-$*)
+	$(call BUILDSTATUS,TIER_FINISH $*)
--- a/client.mk
+++ b/client.mk
@@ -171,17 +171,17 @@ MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C 
 
 endif # MOZ_BUILD_PROJECTS
 
 # 'configure' scripts generated by autoconf.
 CONFIGURES := $(TOPSRCDIR)/configure
 CONFIGURES += $(TOPSRCDIR)/js/src/configure
 
 # Make targets that are going to be passed to the real build system
-OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check
+OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check automation/build
 
 #######################################################################
 # Rules
 
 # The default rule is build
 build::
 	$(MAKE) -f $(TOPSRCDIR)/client.mk $(if $(MOZ_PGO),profiledbuild,realbuild)
 
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -60,17 +60,17 @@ class TierStatus(object):
 
     The build system is organized into linear phases called tiers. Each tier
     executes in the order it was defined, 1 at a time.
     """
 
     def __init__(self, resources):
         """Accepts a SystemResourceMonitor to record results against."""
         self.tiers = OrderedDict()
-        self.active_tier = None
+        self.active_tiers = set()
         self.resources = resources
 
     def set_tiers(self, tiers):
         """Record the set of known tiers."""
         for tier in tiers:
             self.tiers[tier] = dict(
                 begin_time=None,
                 finish_time=None,
@@ -79,27 +79,28 @@ class TierStatus(object):
 
     def begin_tier(self, tier):
         """Record that execution of a tier has begun."""
         t = self.tiers[tier]
         # We should ideally use a monotonic clock here. Unfortunately, we won't
         # have one until Python 3.
         t['begin_time'] = time.time()
         self.resources.begin_phase(tier)
-        self.active_tier = tier
+        self.active_tiers.add(tier)
 
     def finish_tier(self, tier):
         """Record that execution of a tier has finished."""
         t = self.tiers[tier]
         t['finish_time'] = time.time()
         t['duration'] = self.resources.finish_phase(tier)
+        self.active_tiers.remove(tier)
 
     def tier_status(self):
         for tier, state in self.tiers.items():
-            active = self.active_tier == tier
+            active = tier in self.active_tiers
             finished = state['finish_time'] is not None
 
             yield tier, active, finished
 
     def tiered_resource_usage(self):
         """Obtains an object containing resource usage for tiers.
 
         The returned object is suitable for serialization.
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -362,16 +362,28 @@ class Build(MachCommandBase):
                         break
             else:
                 monitor.start_resource_recording()
                 status = self._run_make(srcdir=True, filename='client.mk',
                     line_handler=output.on_line, log=False, print_directory=False,
                     allow_parallel=False, ensure_exit_code=False, num_jobs=jobs,
                     silent=not verbose, force_pymake=pymake)
 
+                make_extra = self.mozconfig['make_extra'] or []
+                make_extra = dict(m.split('=', 1) for m in make_extra)
+
+                moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get('export MOZ_AUTOMATION', None)
+                if moz_automation and status == 0:
+                    # Default to 1 job until we have make-4.0 in for output
+                    # buffering.
+                    status = self._run_make(target='automation/build',
+                        line_handler=output.on_line, log=False, print_directory=False,
+                        allow_parallel=False, ensure_exit_code=False, num_jobs=1,
+                        silent=not verbose, force_pymake=pymake)
+
                 self.log(logging.WARNING, 'warning_summary',
                     {'count': len(monitor.warnings_database)},
                     '{count} compiler warnings present.')
 
             monitor.finish(record_usage=status==0)
 
         high_finder, finder_percent = monitor.have_high_finder_usage()
         if high_finder: