Bug 978211 - enable MOZ_AUTOMATION_* flags in linux mozconfigs; r=glandium
authorMike Shal <mshal@mozilla.com>
Mon, 16 Jun 2014 13:17:21 -0400
changeset 189901 26b95471c5f8f41dfcb207c9358ae70d0f7ad434
parent 189900 c6426f08a82875f128b25aa1c43624b4e99ecf0f
child 189902 a6b5b538815561ac3acdb37a3575a64e35c2ee8f
push id26999
push userryanvm@gmail.com
push dateMon, 23 Jun 2014 14:42:40 +0000
treeherdermozilla-central@4a7f4ed3f08b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs978211
milestone33.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 978211 - enable MOZ_AUTOMATION_* flags in linux mozconfigs; r=glandium
browser/config/mozconfigs/linux32/debug
browser/config/mozconfigs/linux64/debug
browser/config/mozconfigs/linux64/debug-static-analysis-clang
browser/config/mozconfigs/linux64/nightly-nonunified
build/gen_mach_buildprops.py
build/moz-automation.mk
build/mozconfig.automation
build/mozconfig.common
build/unix/mozconfig.asan
build/unix/mozconfig.linux
--- a/browser/config/mozconfigs/linux32/debug
+++ b/browser/config/mozconfigs/linux32/debug
@@ -1,14 +1,17 @@
 ac_add_options --enable-debug
 ac_add_options --enable-dmd
 ac_add_options --enable-signmar
 
 ac_add_options --disable-unified-compilation
 
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_L10N_CHECK=0
+
 . $topsrcdir/build/unix/mozconfig.linux32
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 #Use ccache
 . "$topsrcdir/build/mozconfig.cache"
 
--- a/browser/config/mozconfigs/linux64/debug
+++ b/browser/config/mozconfigs/linux64/debug
@@ -1,12 +1,15 @@
 ac_add_options --enable-debug
 ac_add_options --enable-dmd
 ac_add_options --enable-signmar
 
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_L10N_CHECK=0
+
 . $topsrcdir/build/unix/mozconfig.linux
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 # Use ccache
 . "$topsrcdir/build/mozconfig.cache"
 
--- a/browser/config/mozconfigs/linux64/debug-static-analysis-clang
+++ b/browser/config/mozconfigs/linux64/debug-static-analysis-clang
@@ -1,8 +1,15 @@
+MOZ_AUTOMATION_BUILD_SYMBOLS=0
+MOZ_AUTOMATION_PACKAGE_TESTS=0
+MOZ_AUTOMATION_UNITTEST_SENDCHANGE=0
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_L10N_CHECK=0
+MOZ_AUTOMATION_CHECK=0
+
 . "$topsrcdir/build/mozconfig.common"
 
 ac_add_options --enable-debug
 
 ac_add_options --disable-unified-compilation
 
 # Use Clang as specified in manifest
 export CC="$topsrcdir/clang/bin/clang"
--- a/browser/config/mozconfigs/linux64/nightly-nonunified
+++ b/browser/config/mozconfigs/linux64/nightly-nonunified
@@ -1,3 +1,8 @@
+MOZ_AUTOMATION_UPLOAD=0
+MOZ_AUTOMATION_UNITTEST_SENDCHANGE=0
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_CHECK=0
+
 . "$topsrcdir/browser/config/mozconfigs/linux64/nightly"
 
 ac_add_options --disable-unified-compilation
--- a/build/gen_mach_buildprops.py
+++ b/build/gen_mach_buildprops.py
@@ -1,15 +1,20 @@
 #!/usr/bin/python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import sys, os, sha, json, re
+import sys
+import os
+import sha
+import json
+import re
+import errno
 from argparse import ArgumentParser
 
 def getFileHashAndSize(filename):
     sha1Hash = 'UNKNOWN'
     size = 'UNKNOWN'
 
     try:
         # open in binary mode to make sure we get consistent results
@@ -46,25 +51,30 @@ def getUrlProperties(filename):
         ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
         ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
         # packageUrl must be last!
         ('packageUrl', lambda m: True),
     ]
     url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
     properties = {}
 
-    with open(filename) as f:
-        for line in f:
-            m = url_re.match(line)
-            if m:
-                m = m.group(1)
-                for prop, condition in property_conditions:
-                    if condition(m):
-                        properties.update({prop: m})
-                        break
+    try:
+        with open(filename) as f:
+            for line in f:
+                m = url_re.match(line)
+                if m:
+                    m = m.group(1)
+                    for prop, condition in property_conditions:
+                        if condition(m):
+                            properties.update({prop: m})
+                            break
+    except IOError as e:
+        if e.errno != errno.ENOENT:
+            raise
+        properties = {prop: 'UNKNOWN' for prop, condition in property_conditions}
     return properties
 
 if __name__ == '__main__':
     parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
     parser.add_argument("--complete-mar-file", required=True,
                         action="store", dest="complete_mar_file",
                         help="Path to the complete MAR file, relative to the objdir.")
     parser.add_argument("--upload-output", required=True,
--- a/build/moz-automation.mk
+++ b/build/moz-automation.mk
@@ -12,63 +12,67 @@ endif
 endif
 
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 # Log file from the 'make upload' step. We need this to parse out the URLs of
 # the uploaded files.
 AUTOMATION_UPLOAD_OUTPUT = $(DIST)/automation-upload.txt
 
+# Helper variables to convert from MOZ_AUTOMATION_* variables to the
+# corresponding the make target
+tier_BUILD_SYMBOLS = buildsymbols
+tier_CHECK = check
+tier_L10N_CHECK = l10n-check
+tier_PACKAGE = package
+tier_PACKAGE_TESTS = package-tests
+tier_UPDATE_PACKAGING = update-packaging
+tier_UPLOAD_SYMBOLS = uploadsymbols
+tier_UPLOAD = upload
+
 # Automation build steps. Everything in MOZ_AUTOMATION_TIERS also gets used in
 # TIERS for mach display. As such, the MOZ_AUTOMATION_TIERS are roughly sorted
 # here in the order that they will be executed (since mach doesn't know of the
 # dependencies between them).
-
-MOZ_AUTOMATION_TIERS += package-tests
-MOZ_AUTOMATION_TIERS += buildsymbols
-
-ifdef NIGHTLY_BUILD
-MOZ_AUTOMATION_TIERS += uploadsymbols
-endif
-
-ifdef MOZ_UPDATE_PACKAGING
-MOZ_AUTOMATION_TIERS += update-packaging
-automation/upload: automation/update-packaging
-automation/update-packaging: automation/package
-endif
-
-MOZ_AUTOMATION_TIERS += package
-MOZ_AUTOMATION_TIERS += check
-
-ifndef MOZ_ASAN
-MOZ_AUTOMATION_TIERS += l10n-check
-automation/l10n-check: automation/package
-endif
-
-MOZ_AUTOMATION_TIERS += upload
-
-automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
-	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
+moz_automation_symbols = PACKAGE_TESTS BUILD_SYMBOLS UPLOAD_SYMBOLS PACKAGE UPDATE_PACKAGING CHECK L10N_CHECK UPLOAD
+MOZ_AUTOMATION_TIERS := $(foreach sym,$(moz_automation_symbols),$(if $(filter 1,$(MOZ_AUTOMATION_$(sym))),$(tier_$(sym))))
 
 # Dependencies between automation build steps
 automation/uploadsymbols: automation/buildsymbols
 
+automation/update-packaging: automation/package
+
+automation/l10n-check: automation/package
+
 automation/upload: automation/package
 automation/upload: automation/package-tests
 automation/upload: automation/buildsymbols
+automation/upload: automation/update-packaging
 
 # automation/package and automation/check should depend on build (which is
 # implicit due to the way client.mk invokes automation/build), but buildsymbols
 # changes the binaries/libs, and that's what we package/test.
 automation/package: automation/buildsymbols
 automation/check: automation/buildsymbols
 
+automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
+	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
+
 # make check runs with the keep-going flag so we can see all the failures
 AUTOMATION_EXTRA_CMDLINE-check = -k
 
 # We need the log from make upload to grep it for urls in order to set
 # properties.
 AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT)
 
+# The commands only run if the corresponding MOZ_AUTOMATION_* variable is
+# enabled. This means, for example, if we enable MOZ_AUTOMATION_UPLOAD, then
+# 'buildsymbols' will only run if MOZ_AUTOMATION_BUILD_SYMBOLS is also set.
+# However, the target automation/buildsymbols will still be executed in this
+# case because it is a prerequisite of automation/upload.
+define automation_commands
+$(call BUILDSTATUS,TIER_START $1)
+@$(MAKE) $1 $(AUTOMATION_EXTRA_CMDLINE-$1)
+$(call BUILDSTATUS,TIER_FINISH $1)
+endef
+
 automation/%:
-	$(call BUILDSTATUS,TIER_START $*)
-	@$(MAKE) $* $(AUTOMATION_EXTRA_CMDLINE-$*)
-	$(call BUILDSTATUS,TIER_FINISH $*)
+	$(if $(filter $*,$(MOZ_AUTOMATION_TIERS)),$(call automation_commands,$*))
new file mode 100644
--- /dev/null
+++ b/build/mozconfig.automation
@@ -0,0 +1,21 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Common mozconfig for automation builds.
+#
+# We export MOZ_AUTOMATION_* variables here to trigger various steps in
+# automation builds.  For example, if MOZ_AUTOMATION_PACKAGE is set, then the
+# package step will run.  This file contains the default settings, which can be
+# overridden by setting them earlier in the appropriate mozconfig.
+
+mk_add_options "export MOZ_AUTOMATION_BUILD_SYMBOLS=${MOZ_AUTOMATION_BUILD_SYMBOLS-1}"
+mk_add_options "export MOZ_AUTOMATION_CHECK=${MOZ_AUTOMATION_CHECK-1}"
+mk_add_options "export MOZ_AUTOMATION_L10N_CHECK=${MOZ_AUTOMATION_L10N_CHECK-1}"
+mk_add_options "export MOZ_AUTOMATION_PACKAGE=${MOZ_AUTOMATION_PACKAGE-1}"
+mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=${MOZ_AUTOMATION_PACKAGE_TESTS-1}"
+mk_add_options "export MOZ_AUTOMATION_TALOS_SENDCHANGE=${MOZ_AUTOMATION_TALOS_SENDCHANGE-1}"
+mk_add_options "export MOZ_AUTOMATION_UNITTEST_SENDCHANGE=${MOZ_AUTOMATION_UNITTEST_SENDCHANGE-1}"
+mk_add_options "export MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-0}"
+mk_add_options "export MOZ_AUTOMATION_UPLOAD=${MOZ_AUTOMATION_UPLOAD-1}"
+mk_add_options "export MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-0}"
--- a/build/mozconfig.common
+++ b/build/mozconfig.common
@@ -10,8 +10,10 @@
 # another mozconfig file, you'll need to use mozconfig.common.override instead
 # of this file.
 
 mk_add_options AUTOCLOBBER=1
 
 ac_add_options --enable-crashreporter
 
 ac_add_options --enable-release
+
+. "$topsrcdir/build/mozconfig.automation"
--- a/build/unix/mozconfig.asan
+++ b/build/unix/mozconfig.asan
@@ -1,8 +1,14 @@
+if [ "x$IS_NIGHTLY" = "xyes" ]; then
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_L10N_CHECK=0
+
 . "$topsrcdir/build/mozconfig.common"
 
 # Use Clang as specified in manifest
 export CC="$topsrcdir/clang/bin/clang -fgnu89-inline"
 export CXX="$topsrcdir/clang/bin/clang++"
 export LLVM_SYMBOLIZER="$topsrcdir/clang/bin/llvm-symbolizer"
 
 # Mandatory flags for ASan
--- a/build/unix/mozconfig.linux
+++ b/build/unix/mozconfig.linux
@@ -1,8 +1,13 @@
+if [ "x$IS_NIGHTLY" = "xyes" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=1
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/build/mozconfig.common"
 
 # some b2g desktop builds still happen on i686 machines, and the tooltool
 # toolchain is x86_64 only.
 # We also deal with valgrind builds here, they don't use tooltool manifests at
 # all yet.
 if [ -z "$no_tooltool" ]
 then