Bug 1162171: Split out source checkouts into its own action for fx_desktop_build r=mshal
author"Chris AtLee"
Wed, 06 May 2015 16:27:47 -0400 (2015-05-06)
changeset 3978 a8140ebac3b5198bff045a0da2e88638f4ad8bbc
parent 3977 aea5cb0e963078b82df894131a1661ec0fe844d8
child 3979 28b10345448dafe13f79780b6a775fe4b9d8f55d
child 3980 9f743f3621a08b5ef9da3148c6b45395fccae43c
push id3150
push usercatlee@mozilla.com
push dateWed, 06 May 2015 20:28:08 +0000 (2015-05-06)
reviewersmshal
bugs1162171
Bug 1162171: Split out source checkouts into its own action for fx_desktop_build r=mshal
configs/builds/releng_base_android_64_builds.py
configs/builds/releng_base_linux_32_builds.py
configs/builds/releng_base_linux_64_builds.py
configs/builds/releng_base_mac_64_builds.py
configs/builds/releng_base_windows_32_builds.py
configs/builds/releng_base_windows_64_builds.py
configs/builds/releng_sub_linux_configs/32_debug.py
configs/builds/releng_sub_linux_configs/64_asan.py
configs/builds/releng_sub_linux_configs/64_asan_and_debug.py
configs/builds/releng_sub_linux_configs/64_code_coverage.py
configs/builds/releng_sub_linux_configs/64_debug.py
configs/builds/releng_sub_linux_configs/64_graphene.py
configs/builds/releng_sub_linux_configs/64_mulet.py
configs/builds/releng_sub_linux_configs/64_stat_and_debug.py
configs/builds/releng_sub_mac_configs/64_debug.py
configs/builds/releng_sub_mac_configs/64_graphene.py
configs/builds/releng_sub_mac_configs/64_mulet.py
configs/builds/releng_sub_mac_configs/64_stat_and_debug.py
configs/builds/releng_sub_windows_configs/32_debug.py
configs/builds/releng_sub_windows_configs/32_mulet.py
configs/builds/releng_sub_windows_configs/64_debug.py
configs/builds/releng_sub_windows_configs/64_graphene.py
mozharness/mozilla/building/buildbase.py
scripts/fx_desktop_build.py
--- a/configs/builds/releng_base_android_64_builds.py
+++ b/configs/builds/releng_base_android_64_builds.py
@@ -2,16 +2,17 @@ import os
 
 config = {
     #########################################################################
     ######## ANDROID GENERIC CONFIG KEYS/VAlUES
 
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         'multi-l10n',
     ],
     "buildbot_json_path": "buildprops.json",
     'exes': {
         'hgtool.py': os.path.join(
             os.getcwd(), 'build', 'tools', 'buildfarm', 'utils', 'hgtool.py'
--- a/configs/builds/releng_base_linux_32_builds.py
+++ b/configs/builds/releng_base_linux_32_builds.py
@@ -6,16 +6,17 @@ config = {
     # if you are updating this with custom 32 bit keys/values please add them
     # below under the '32 bit specific' code block otherwise, update in this
     # code block and also make sure this is synced with
     # releng_base_linux_64_builds.py
 
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     "buildbot_json_path": "buildprops.json",
     'exes': {
         'hgtool.py': os.path.join(
--- a/configs/builds/releng_base_linux_64_builds.py
+++ b/configs/builds/releng_base_linux_64_builds.py
@@ -6,16 +6,17 @@ config = {
     # if you are updating this with custom 64 bit keys/values please add them
     # below under the '64 bit specific' code block otherwise, update in this
     # code block and also make sure this is synced with
     # releng_base_linux_64_builds.py
 
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     "buildbot_json_path": "buildprops.json",
     'exes': {
         'hgtool.py': os.path.join(
--- a/configs/builds/releng_base_mac_64_builds.py
+++ b/configs/builds/releng_base_mac_64_builds.py
@@ -4,16 +4,17 @@ import sys
 config = {
     #########################################################################
     ######## MACOSX GENERIC CONFIG KEYS/VAlUES
 
     'default_actions': [
         'clobber',
         'clone-tools',
         # 'setup-mock',
+        'checkout-sources',
         'build',
         'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     "buildbot_json_path": "buildprops.json",
     'exes': {
         'python2.7': sys.executable,
         'hgtool.py': os.path.join(
--- a/configs/builds/releng_base_windows_32_builds.py
+++ b/configs/builds/releng_base_windows_32_builds.py
@@ -7,16 +7,17 @@ config = {
     # if you are updating this with custom 32 bit keys/values please add them
     # below under the '32 bit specific' code block otherwise, update in this
     # code block and also make sure this is synced with
     # releng_base_windows_64_builds.py
 
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         # 'setup-mock', windows do not use mock
         'build',
         'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     "buildbot_json_path": "buildprops.json",
     'exes': {
         'python2.7': sys.executable,
--- a/configs/builds/releng_base_windows_64_builds.py
+++ b/configs/builds/releng_base_windows_64_builds.py
@@ -7,16 +7,17 @@ config = {
     # if you are updating this with custom 32 bit keys/values please add them
     # below under the '32 bit specific' code block otherwise, update in this
     # code block and also make sure this is synced with
     # releng_base_windows_64_builds.py
 
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         # 'setup-mock', windows do not use mock
         'build',
         'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     "buildbot_json_path": "buildprops.json",
     'exes': {
         'python2.7': sys.executable,
--- a/configs/builds/releng_sub_linux_configs/32_debug.py
+++ b/configs/builds/releng_sub_linux_configs/32_debug.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         # 'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     'debug_build': True,
     'stage_platform': 'linux-debug',
     'purge_minsize': 15,
--- a/configs/builds/releng_sub_linux_configs/64_asan.py
+++ b/configs/builds/releng_sub_linux_configs/64_asan.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         # 'generate-build-stats',
         # 'update',
     ],
     'stage_platform': 'linux64-asan',
     'purge_minsize': 12,
     'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
--- a/configs/builds/releng_sub_linux_configs/64_asan_and_debug.py
+++ b/configs/builds/releng_sub_linux_configs/64_asan_and_debug.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         # 'generate-build-stats',
         # 'update',
     ],
     'stage_platform': 'linux64-asan-debug',
     'debug_build': True,
     'purge_minsize': 12,
--- a/configs/builds/releng_sub_linux_configs/64_code_coverage.py
+++ b/configs/builds/releng_sub_linux_configs/64_code_coverage.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         # 'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     'stage_platform': 'linux64-cc',
     'platform_supports_post_upload_to_latest': False,
     'enable_signing': False,
--- a/configs/builds/releng_sub_linux_configs/64_debug.py
+++ b/configs/builds/releng_sub_linux_configs/64_debug.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         # 'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     'stage_platform': 'linux64-debug',
     'debug_build': True,
     'enable_signing': False,
--- a/configs/builds/releng_sub_linux_configs/64_graphene.py
+++ b/configs/builds/releng_sub_linux_configs/64_graphene.py
@@ -1,14 +1,15 @@
 import os
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         'update',  # may or may not happen based on query_is_nightly()
     ],
     'stage_platform': 'linux64',
     'stage_product': 'b2g',
     'enable_signing': False,
     'enable_talos_sendchange': False,
--- a/configs/builds/releng_sub_linux_configs/64_mulet.py
+++ b/configs/builds/releng_sub_linux_configs/64_mulet.py
@@ -1,14 +1,15 @@
 import os
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
     ],
     'stage_platform': 'linux64-mulet',
     'stage_product': 'b2g',
     'tooltool_manifest_src': "b2g/dev/config/tooltool-manifests/linux64/releng.manifest",
     'platform_supports_post_upload_to_latest': False,
     'enable_signing': False,
--- a/configs/builds/releng_sub_linux_configs/64_stat_and_debug.py
+++ b/configs/builds/releng_sub_linux_configs/64_stat_and_debug.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'setup-mock',
         'build',
         # 'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     'debug_build': True,
     'stage_platform': 'linux64-st-an-debug',
     'purge_minsize': 12,
--- a/configs/builds/releng_sub_mac_configs/64_debug.py
+++ b/configs/builds/releng_sub_mac_configs/64_debug.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         # 'setup-mock',
         'build',
         # 'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     'stage_platform': 'macosx64-debug',
     'debug_build': True,
     'objdir': 'obj-firefox',
--- a/configs/builds/releng_sub_mac_configs/64_graphene.py
+++ b/configs/builds/releng_sub_mac_configs/64_graphene.py
@@ -1,14 +1,15 @@
 import os
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'build',
         'update',  # may or may not happen based on query_is_nightly()
     ],
     'stage_platform': 'mac',
     'stage_product': 'b2g',
     'enable_signing': False,
     'enable_talos_sendchange': False,
     'enable_unittest_sendchange': False,
--- a/configs/builds/releng_sub_mac_configs/64_mulet.py
+++ b/configs/builds/releng_sub_mac_configs/64_mulet.py
@@ -1,14 +1,15 @@
 import os
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'build',
     ],
     'stage_platform': 'macosx64-mulet',
     'stage_product': 'b2g',
     'tooltool_manifest_src': "b2g/dev/config/tooltool-manifests/macosx64/releng.manifest",
     'objdir': 'obj-firefox',
     'enable_signing': False,
     'enable_unittest_sendchange': False,
--- a/configs/builds/releng_sub_mac_configs/64_stat_and_debug.py
+++ b/configs/builds/releng_sub_mac_configs/64_stat_and_debug.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         # 'setup-mock',
         'build',
         # 'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     'debug_build': True,
     'stage_platform': 'macosx64-st-an-debug',
     'purge_minsize': 12,
--- a/configs/builds/releng_sub_windows_configs/32_debug.py
+++ b/configs/builds/releng_sub_windows_configs/32_debug.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         # 'setup-mock', windows do not use mock
         'build',
         # 'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     'stage_platform': 'win32-debug',
     'debug_build': True,
     'enable_talos_sendchange': False,
--- a/configs/builds/releng_sub_windows_configs/32_mulet.py
+++ b/configs/builds/releng_sub_windows_configs/32_mulet.py
@@ -1,14 +1,15 @@
 import os
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'build',
     ],
     'stage_platform': 'win32-mulet',
     'stage_product': 'b2g',
     'tooltool_manifest_src': "b2g/dev/config/tooltool-manifests/win32/releng.manifest",
     'enable_signing': False,
     'enable_talos_sendchange': False,
     'enable_unittest_sendchange': False,
--- a/configs/builds/releng_sub_windows_configs/64_debug.py
+++ b/configs/builds/releng_sub_windows_configs/64_debug.py
@@ -1,16 +1,17 @@
 import os
 
 MOZ_OBJDIR = 'obj-firefox'
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         # 'setup-mock', windows do not use mock
         'build',
         # 'generate-build-stats',
         'update',  # decided by query_is_nightly()
     ],
     'stage_platform': 'win64-debug',
     'debug_build': True,
     'enable_talos_sendchange': False,
--- a/configs/builds/releng_sub_windows_configs/64_graphene.py
+++ b/configs/builds/releng_sub_windows_configs/64_graphene.py
@@ -1,14 +1,15 @@
 import os
 
 config = {
     'default_actions': [
         'clobber',
         'clone-tools',
+        'checkout-sources',
         'build',
         'update',  # may or may not happen based on query_is_nightly()
     ],
     'stage_platform': 'win64',
     'stage_product': 'b2g',
     'enable_signing': False,
     'enable_talos_sendchange': False,
     'enable_unittest_sendchange': False,
--- a/mozharness/mozilla/building/buildbase.py
+++ b/mozharness/mozilla/building/buildbase.py
@@ -22,17 +22,17 @@ import glob
 import logging
 from itertools import chain
 
 # import the power of mozharness ;)
 import sys
 from datetime import datetime
 import re
 from mozharness.base.config import BaseConfig, parse_config_file
-from mozharness.base.log import ERROR, OutputParser, FATAL, WARNING
+from mozharness.base.log import ERROR, OutputParser, FATAL
 from mozharness.base.script import PostScriptRun
 from mozharness.base.transfer import TransferMixin
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.buildbot import BuildbotMixin, TBPL_STATUS_DICT, \
     TBPL_EXCEPTION, TBPL_RETRY, EXIT_STATUS_DICT, TBPL_WARNING, TBPL_SUCCESS, \
     TBPL_WORST_LEVEL_TUPLE, TBPL_FAILURE
 from mozharness.mozilla.purge import PurgeMixin
 from mozharness.mozilla.mock import MockMixin
@@ -1511,27 +1511,29 @@ or run without that action (ie: --no-{ac
             env = self.query_build_env()
             mozbuild_path = env.get('MOZBUILD_STATE_PATH')
         if mozbuild_path:
             self.mkdir_p(mozbuild_path)
         else:
             self.warning("mozbuild_path could not be determined. skipping "
                          "creating it.")
 
+    def checkout_sources(self):
+        self._checkout_source()
+
     def preflight_build(self):
         """set up machine state for a complete build."""
         c = self.config
         if c.get('enable_ccache'):
             self._ccache_z()
         if not self.query_is_nightly():
             # the old package should live in source dir so we don't need to do
             # this for nighties since we clobber the whole work_dir in
             # clobber()
             self._rm_old_package()
-        self._checkout_source()
         self._get_mozconfig()
         self._run_tooltool()
         self._create_mozbuild_dir()
         mach_props = os.path.join(
             self.query_abs_dirs()['abs_obj_dir'], 'mach_build_properties.json'
         )
         if os.path.exists(mach_props):
             self.info("Removing previous mach property file: %s" % mach_props)
--- a/scripts/fx_desktop_build.py
+++ b/scripts/fx_desktop_build.py
@@ -25,16 +25,17 @@ from mozharness.mozilla.building.buildba
 
 class FxDesktopBuild(BuildScript, object):
     def __init__(self):
         buildscript_kwargs = {
             'config_options': BUILD_BASE_CONFIG_OPTIONS,
             'all_actions': [
                 'clobber',
                 'clone-tools',
+                'checkout-sources',
                 'setup-mock',
                 'build',
                 'package-source',
                 'multi-l10n',
                 'generate-build-stats',
                 'update',
             ],
             'require_config_file': True,