Bug 978211 - enable MOZ_AUTOMATION_* flags in linux mozconfigs; r=glandium
authorMike Shal <mshal@mozilla.com>
Mon, 16 Jun 2014 13:17:21 -0400
changeset 189907 26b95471c5f8f41dfcb207c9358ae70d0f7ad434
parent 189906 c6426f08a82875f128b25aa1c43624b4e99ecf0f
child 189908 a6b5b538815561ac3acdb37a3575a64e35c2ee8f
push id7399
push userryanvm@gmail.com
push dateMon, 23 Jun 2014 14:59:50 +0000
treeherderfx-team@007a3b5039a4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs978211
milestone33.0a1
Bug 978211 - enable MOZ_AUTOMATION_* flags in linux mozconfigs; r=glandium
browser/config/mozconfigs/linux32/debug
browser/config/mozconfigs/linux64/debug
browser/config/mozconfigs/linux64/debug-static-analysis-clang
browser/config/mozconfigs/linux64/nightly-nonunified
build/gen_mach_buildprops.py
build/moz-automation.mk
build/mozconfig.automation
build/mozconfig.common
build/unix/mozconfig.asan
build/unix/mozconfig.linux
--- a/browser/config/mozconfigs/linux32/debug
+++ b/browser/config/mozconfigs/linux32/debug
@@ -1,14 +1,17 @@
 ac_add_options --enable-debug
 ac_add_options --enable-dmd
 ac_add_options --enable-signmar
 
 ac_add_options --disable-unified-compilation
 
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_L10N_CHECK=0
+
 . $topsrcdir/build/unix/mozconfig.linux32
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 #Use ccache
 . "$topsrcdir/build/mozconfig.cache"
 
--- a/browser/config/mozconfigs/linux64/debug
+++ b/browser/config/mozconfigs/linux64/debug
@@ -1,12 +1,15 @@
 ac_add_options --enable-debug
 ac_add_options --enable-dmd
 ac_add_options --enable-signmar
 
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_L10N_CHECK=0
+
 . $topsrcdir/build/unix/mozconfig.linux
 
 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
 
 # Use ccache
 . "$topsrcdir/build/mozconfig.cache"
 
--- a/browser/config/mozconfigs/linux64/debug-static-analysis-clang
+++ b/browser/config/mozconfigs/linux64/debug-static-analysis-clang
@@ -1,8 +1,15 @@
+MOZ_AUTOMATION_BUILD_SYMBOLS=0
+MOZ_AUTOMATION_PACKAGE_TESTS=0
+MOZ_AUTOMATION_UNITTEST_SENDCHANGE=0
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_L10N_CHECK=0
+MOZ_AUTOMATION_CHECK=0
+
 . "$topsrcdir/build/mozconfig.common"
 
 ac_add_options --enable-debug
 
 ac_add_options --disable-unified-compilation
 
 # Use Clang as specified in manifest
 export CC="$topsrcdir/clang/bin/clang"
--- a/browser/config/mozconfigs/linux64/nightly-nonunified
+++ b/browser/config/mozconfigs/linux64/nightly-nonunified
@@ -1,3 +1,8 @@
+MOZ_AUTOMATION_UPLOAD=0
+MOZ_AUTOMATION_UNITTEST_SENDCHANGE=0
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_CHECK=0
+
 . "$topsrcdir/browser/config/mozconfigs/linux64/nightly"
 
 ac_add_options --disable-unified-compilation
--- a/build/gen_mach_buildprops.py
+++ b/build/gen_mach_buildprops.py
@@ -1,15 +1,20 @@
 #!/usr/bin/python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import sys, os, sha, json, re
+import sys
+import os
+import sha
+import json
+import re
+import errno
 from argparse import ArgumentParser
 
 def getFileHashAndSize(filename):
     sha1Hash = 'UNKNOWN'
     size = 'UNKNOWN'
 
     try:
         # open in binary mode to make sure we get consistent results
@@ -46,25 +51,30 @@ def getUrlProperties(filename):
         ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
         ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
         # packageUrl must be last!
         ('packageUrl', lambda m: True),
     ]
     url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
     properties = {}
 
-    with open(filename) as f:
-        for line in f:
-            m = url_re.match(line)
-            if m:
-                m = m.group(1)
-                for prop, condition in property_conditions:
-                    if condition(m):
-                        properties.update({prop: m})
-                        break
+    try:
+        with open(filename) as f:
+            for line in f:
+                m = url_re.match(line)
+                if m:
+                    m = m.group(1)
+                    for prop, condition in property_conditions:
+                        if condition(m):
+                            properties.update({prop: m})
+                            break
+    except IOError as e:
+        if e.errno != errno.ENOENT:
+            raise
+        properties = {prop: 'UNKNOWN' for prop, condition in property_conditions}
     return properties
 
 if __name__ == '__main__':
     parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
     parser.add_argument("--complete-mar-file", required=True,
                         action="store", dest="complete_mar_file",
                         help="Path to the complete MAR file, relative to the objdir.")
     parser.add_argument("--upload-output", required=True,
--- a/build/moz-automation.mk
+++ b/build/moz-automation.mk
@@ -12,63 +12,67 @@ endif
 endif
 
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 # Log file from the 'make upload' step. We need this to parse out the URLs of
 # the uploaded files.
 AUTOMATION_UPLOAD_OUTPUT = $(DIST)/automation-upload.txt
 
+# Helper variables to convert from MOZ_AUTOMATION_* variables to the
+# corresponding the make target
+tier_BUILD_SYMBOLS = buildsymbols
+tier_CHECK = check
+tier_L10N_CHECK = l10n-check
+tier_PACKAGE = package
+tier_PACKAGE_TESTS = package-tests
+tier_UPDATE_PACKAGING = update-packaging
+tier_UPLOAD_SYMBOLS = uploadsymbols
+tier_UPLOAD = upload
+
 # Automation build steps. Everything in MOZ_AUTOMATION_TIERS also gets used in
 # TIERS for mach display. As such, the MOZ_AUTOMATION_TIERS are roughly sorted
 # here in the order that they will be executed (since mach doesn't know of the
 # dependencies between them).
-
-MOZ_AUTOMATION_TIERS += package-tests
-MOZ_AUTOMATION_TIERS += buildsymbols
-
-ifdef NIGHTLY_BUILD
-MOZ_AUTOMATION_TIERS += uploadsymbols
-endif
-
-ifdef MOZ_UPDATE_PACKAGING
-MOZ_AUTOMATION_TIERS += update-packaging
-automation/upload: automation/update-packaging
-automation/update-packaging: automation/package
-endif
-
-MOZ_AUTOMATION_TIERS += package
-MOZ_AUTOMATION_TIERS += check
-
-ifndef MOZ_ASAN
-MOZ_AUTOMATION_TIERS += l10n-check
-automation/l10n-check: automation/package
-endif
-
-MOZ_AUTOMATION_TIERS += upload
-
-automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
-	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
+moz_automation_symbols = PACKAGE_TESTS BUILD_SYMBOLS UPLOAD_SYMBOLS PACKAGE UPDATE_PACKAGING CHECK L10N_CHECK UPLOAD
+MOZ_AUTOMATION_TIERS := $(foreach sym,$(moz_automation_symbols),$(if $(filter 1,$(MOZ_AUTOMATION_$(sym))),$(tier_$(sym))))
 
 # Dependencies between automation build steps
 automation/uploadsymbols: automation/buildsymbols
 
+automation/update-packaging: automation/package
+
+automation/l10n-check: automation/package
+
 automation/upload: automation/package
 automation/upload: automation/package-tests
 automation/upload: automation/buildsymbols
+automation/upload: automation/update-packaging
 
 # automation/package and automation/check should depend on build (which is
 # implicit due to the way client.mk invokes automation/build), but buildsymbols
 # changes the binaries/libs, and that's what we package/test.
 automation/package: automation/buildsymbols
 automation/check: automation/buildsymbols
 
+automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
+	$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
+
 # make check runs with the keep-going flag so we can see all the failures
 AUTOMATION_EXTRA_CMDLINE-check = -k
 
 # We need the log from make upload to grep it for urls in order to set
 # properties.
 AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT)
 
+# The commands only run if the corresponding MOZ_AUTOMATION_* variable is
+# enabled. This means, for example, if we enable MOZ_AUTOMATION_UPLOAD, then
+# 'buildsymbols' will only run if MOZ_AUTOMATION_BUILD_SYMBOLS is also set.
+# However, the target automation/buildsymbols will still be executed in this
+# case because it is a prerequisite of automation/upload.
+define automation_commands
+$(call BUILDSTATUS,TIER_START $1)
+@$(MAKE) $1 $(AUTOMATION_EXTRA_CMDLINE-$1)
+$(call BUILDSTATUS,TIER_FINISH $1)
+endef
+
 automation/%:
-	$(call BUILDSTATUS,TIER_START $*)
-	@$(MAKE) $* $(AUTOMATION_EXTRA_CMDLINE-$*)
-	$(call BUILDSTATUS,TIER_FINISH $*)
+	$(if $(filter $*,$(MOZ_AUTOMATION_TIERS)),$(call automation_commands,$*))
new file mode 100644
--- /dev/null
+++ b/build/mozconfig.automation
@@ -0,0 +1,21 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Common mozconfig for automation builds.
+#
+# We export MOZ_AUTOMATION_* variables here to trigger various steps in
+# automation builds.  For example, if MOZ_AUTOMATION_PACKAGE is set, then the
+# package step will run.  This file contains the default settings, which can be
+# overridden by setting them earlier in the appropriate mozconfig.
+
+mk_add_options "export MOZ_AUTOMATION_BUILD_SYMBOLS=${MOZ_AUTOMATION_BUILD_SYMBOLS-1}"
+mk_add_options "export MOZ_AUTOMATION_CHECK=${MOZ_AUTOMATION_CHECK-1}"
+mk_add_options "export MOZ_AUTOMATION_L10N_CHECK=${MOZ_AUTOMATION_L10N_CHECK-1}"
+mk_add_options "export MOZ_AUTOMATION_PACKAGE=${MOZ_AUTOMATION_PACKAGE-1}"
+mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=${MOZ_AUTOMATION_PACKAGE_TESTS-1}"
+mk_add_options "export MOZ_AUTOMATION_TALOS_SENDCHANGE=${MOZ_AUTOMATION_TALOS_SENDCHANGE-1}"
+mk_add_options "export MOZ_AUTOMATION_UNITTEST_SENDCHANGE=${MOZ_AUTOMATION_UNITTEST_SENDCHANGE-1}"
+mk_add_options "export MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-0}"
+mk_add_options "export MOZ_AUTOMATION_UPLOAD=${MOZ_AUTOMATION_UPLOAD-1}"
+mk_add_options "export MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-0}"
--- a/build/mozconfig.common
+++ b/build/mozconfig.common
@@ -10,8 +10,10 @@
 # another mozconfig file, you'll need to use mozconfig.common.override instead
 # of this file.
 
 mk_add_options AUTOCLOBBER=1
 
 ac_add_options --enable-crashreporter
 
 ac_add_options --enable-release
+
+. "$topsrcdir/build/mozconfig.automation"
--- a/build/unix/mozconfig.asan
+++ b/build/unix/mozconfig.asan
@@ -1,8 +1,14 @@
+if [ "x$IS_NIGHTLY" = "xyes" ]; then
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+MOZ_AUTOMATION_TALOS_SENDCHANGE=0
+MOZ_AUTOMATION_L10N_CHECK=0
+
 . "$topsrcdir/build/mozconfig.common"
 
 # Use Clang as specified in manifest
 export CC="$topsrcdir/clang/bin/clang -fgnu89-inline"
 export CXX="$topsrcdir/clang/bin/clang++"
 export LLVM_SYMBOLIZER="$topsrcdir/clang/bin/llvm-symbolizer"
 
 # Mandatory flags for ASan
--- a/build/unix/mozconfig.linux
+++ b/build/unix/mozconfig.linux
@@ -1,8 +1,13 @@
+if [ "x$IS_NIGHTLY" = "xyes" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=1
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/build/mozconfig.common"
 
 # some b2g desktop builds still happen on i686 machines, and the tooltool
 # toolchain is x86_64 only.
 # We also deal with valgrind builds here, they don't use tooltool manifests at
 # all yet.
 if [ -z "$no_tooltool" ]
 then