Backed out 4 changesets (bug 1415618) for build bustage failures. r=backout on a CLOSED TREE
authorBrindusan Cristian <cbrindusan@mozilla.com>
Thu, 16 Nov 2017 05:53:38 +0200
changeset 392094 b89090531f6a3da0b51d9598c6dc93b1defbbb78
parent 392093 71510fd07ef52f8539ed37968e2ec7ea9c1a5fdd
child 392095 daaf6edb12f7deed6f1c9b74dd3e9e8090f0a1e3
push id32910
push userrgurzau@mozilla.com
push dateThu, 16 Nov 2017 10:02:59 +0000
treeherdermozilla-central@9941e68b5a53 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1415618
milestone59.0a1
backs out71510fd07ef52f8539ed37968e2ec7ea9c1a5fdd
780f67a36d6d91d4ee35070dc7f8d8c598106929
5ecdc55db739801216435854ce412694a8e0dca7
be317382298705c63d0be2f4686eaa7ca513ea6c
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 4 changesets (bug 1415618) for build bustage failures. r=backout on a CLOSED TREE Backed out changeset 71510fd07ef5 (bug 1415618) Backed out changeset 780f67a36d6d (bug 1415618) Backed out changeset 5ecdc55db739 (bug 1415618) Backed out changeset be3173822987 (bug 1415618)
taskcluster/scripts/builder/build-linux.sh
taskcluster/taskgraph/transforms/job/mozharness.py
testing/mozharness/mozharness/base/config.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/taskcluster/scripts/builder/build-linux.sh
+++ b/taskcluster/scripts/builder/build-linux.sh
@@ -7,17 +7,16 @@ echo "running as" $(id)
 ####
 # Taskcluster friendly wrapper for performing fx desktop builds via mozharness.
 ####
 
 # Inputs, with defaults
 
 : MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
 : MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
-: MOZHARNESS_CONFIG_PATHS       ${MOZHARNESS_CONFIG_PATHS}
 : MOZHARNESS_ACTIONS            ${MOZHARNESS_ACTIONS}
 : MOZHARNESS_OPTIONS            ${MOZHARNESS_OPTIONS}
 
 : TOOLTOOL_CACHE                ${TOOLTOOL_CACHE:=/builds/worker/tooltool-cache}
 
 : NEED_XVFB                     ${NEED_XVFB:=false}
 
 : MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
@@ -84,21 +83,16 @@ if [ -n "${MH_CUSTOM_BUILD_VARIANT_CFG}"
     custom_build_variant_cfg_flag="--custom-build-variant-cfg=${MH_CUSTOM_BUILD_VARIANT_CFG}"
 fi
 
 # $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the
 # cache.  However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be
 # entirely effective.
 export TOOLTOOL_CACHE
 
-config_path_cmds=""
-for path in ${MOZHARNESS_CONFIG_PATHS}; do
-    config_path_cmds="${config_path_cmds} --extra-config-path ${WORKSPACE}/build/src/${path}"
-done
-
 # support multiple, space delimited, config files
 config_cmds=""
 for cfg in $MOZHARNESS_CONFIG; do
   config_cmds="${config_cmds} --config ${cfg}"
 done
 
 # if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions
 # in the mozharness configuration)
@@ -115,19 +109,17 @@ if [ -n "$MOZHARNESS_OPTIONS" ]; then
     options=""
     for option in $MOZHARNESS_OPTIONS; do
         options="$options --$option"
     done
 fi
 
 cd /builds/worker
 
-python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} \
-  ${config_path_cmds} \
-  ${config_cmds} \
+python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
   $debug_flag \
   $custom_build_variant_cfg_flag \
   --disable-mock \
   $actions \
   $options \
   --log-level=debug \
   --scm-level=$MOZ_SCM_LEVEL \
   --work-dir=$WORKSPACE/build \
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -29,23 +29,18 @@ from taskgraph.transforms.job.common imp
 
 mozharness_run_schema = Schema({
     Required('using'): 'mozharness',
 
     # the mozharness script used to run this task, relative to the testing/
     # directory and using forward slashes even on Windows
     Required('script'): basestring,
 
-    # Additional paths to look for mozharness configs in. These should be
-    # relative to the base of the source checkout
-    Optional('config-paths'): [basestring],
-
     # the config files required for the task, relative to
-    # testing/mozharness/configs or one of the paths specified in
-    # `config-paths` and using forward slashes even on Windows
+    # testing/mozharness/configs and using forward slashes even on Windows
     Required('config'): [basestring],
 
     # any additional actions to pass to the mozharness command
     Optional('actions'): [basestring],
 
     # any additional options (without leading --) to be passed to mozharness
     Optional('options'): [basestring],
 
@@ -140,19 +135,16 @@ def mozharness_on_docker_worker_setup(co
     })
 
     if 'actions' in run:
         env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
 
     if 'options' in run:
         env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
 
-    if 'config-paths' in run:
-        env['MOZHARNESS_CONFIG_PATHS'] = ' '.join(run['config-paths'])
-
     if 'custom-build-variant-cfg' in run:
         env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
 
     if 'job-script' in run:
         env['JOB_SCRIPT'] = run['job-script']
 
     if 'try' in config.params['project']:
         env['TRY_COMMIT_MSG'] = config.params['message']
@@ -238,22 +230,16 @@ def mozharness_on_generic_worker(config,
 
     if not job['attributes']['build_platform'].startswith('win'):
         raise Exception(
             "Task generation for mozharness build jobs currently only supported on Windows"
         )
 
     mh_command = [r'c:\mozilla-build\python\python.exe']
     mh_command.append('\\'.join([r'.\build\src\testing', run['script'].replace('/', '\\')]))
-
-    if 'config-paths' in run:
-        for path in run['config-paths']:
-            mh_command.append(r'--extra-config-path '
-                              r'.\build\src\{}'.format(path.replace('/', '\\')))
-
     for cfg in run['config']:
         mh_command.append('--config ' + cfg.replace('/', '\\'))
     if run['use-magic-mh-args']:
         mh_command.append('--branch ' + config.params['project'])
         mh_command.append(r'--skip-buildbot-actions')
     mh_command.append(r'--work-dir %cd:Z:=z:%\build')
     for action in run.get('actions', []):
         assert ' ' not in action
--- a/testing/mozharness/mozharness/base/config.py
+++ b/testing/mozharness/mozharness/base/config.py
@@ -130,29 +130,28 @@ class ReadOnlyDict(dict):
         memo[id(self)] = result
         for k, v in self.__dict__.items():
             setattr(result, k, deepcopy(v, memo))
         result._lock = False
         for k, v in self.items():
             result[k] = deepcopy(v, memo)
         return result
 
-DEFAULT_CONFIG_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "configs")
-
 # parse_config_file {{{1
 def parse_config_file(file_name, quiet=False, search_path=None,
                       config_dict_name="config"):
     """Read a config file and return a dictionary.
     """
     file_path = None
     if os.path.exists(file_name):
         file_path = file_name
     else:
         if not search_path:
-            search_path = ['.', DEFAULT_CONFIG_PATH]
+            search_path = ['.', os.path.join(sys.path[0], '..', 'configs'),
+                           os.path.join(sys.path[0], '..', '..', 'configs')]
         for path in search_path:
             if os.path.exists(os.path.join(path, file_name)):
                 file_path = os.path.join(path, file_name)
                 break
         else:
             raise IOError("Can't find %s in %s!" % (file_name, search_path))
     if file_name.endswith('.py'):
         global_dict = {}
@@ -276,20 +275,16 @@ class BaseConfig(object):
             help="Specify the work_dir (subdir of base_work_dir)"
         )
         self.config_parser.add_option(
             "--base-work-dir", action="store", dest="base_work_dir",
             type="string", default=os.getcwd(),
             help="Specify the absolute path of the parent of the working directory"
         )
         self.config_parser.add_option(
-            "--extra-config-path", action='extend', dest="config_paths",
-            type="string", help="Specify additional paths to search for config files.",
-        )
-        self.config_parser.add_option(
             "-c", "--config-file", "--cfg", action="extend", dest="config_files",
             type="string", help="Specify a config file; can be repeated"
         )
         self.config_parser.add_option(
             "-C", "--opt-config-file", "--opt-cfg", action="extend",
             dest="opt_config_files", type="string", default=[],
             help="Specify an optional config file, like --config-file but with no "
                  "error if the file is missing; can be repeated"
@@ -432,31 +427,28 @@ class BaseConfig(object):
         This function is also responsible for downloading any configuration
         files specified by URL.  It uses ``parse_config_file`` in this module
         to parse individual files.
 
         This method can be overridden in a subclass to add extra logic to the
         way that self.config is made up.  See
         `mozharness.mozilla.building.buildbase.BuildingConfig` for an example.
         """
-        config_paths = options.config_paths or ['.']
         all_cfg_files_and_dicts = []
         for cf in all_config_files:
             try:
                 if '://' in cf:  # config file is an url
                     file_name = os.path.basename(cf)
                     file_path = os.path.join(os.getcwd(), file_name)
                     download_config_file(cf, file_path)
                     all_cfg_files_and_dicts.append(
-                        (file_path, parse_config_file(file_path, search_path=["."]))
+                        (file_path, parse_config_file(file_path))
                     )
                 else:
-                    all_cfg_files_and_dicts.append(
-                        (cf, parse_config_file(cf, search_path=config_paths + [DEFAULT_CONFIG_PATH]))
-                    )
+                    all_cfg_files_and_dicts.append((cf, parse_config_file(cf)))
             except Exception:
                 if cf in options.opt_config_files:
                     print(
                         "WARNING: optional config file not found %s" % cf
                     )
                 else:
                     raise
         return all_cfg_files_and_dicts
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -21,17 +21,17 @@ import copy
 import glob
 import shlex
 from itertools import chain
 
 # import the power of mozharness ;)
 import sys
 from datetime import datetime
 import re
-from mozharness.base.config import BaseConfig, parse_config_file, DEFAULT_CONFIG_PATH
+from mozharness.base.config import BaseConfig, parse_config_file
 from mozharness.base.log import ERROR, OutputParser, FATAL
 from mozharness.base.script import PostScriptRun
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.buildbot import (
     BuildbotMixin,
     EXIT_STATUS_DICT,
     TBPL_STATUS_DICT,
     TBPL_EXCEPTION,
@@ -394,16 +394,20 @@ class BuildingConfig(BaseConfig):
         return all_config_dicts
 
 
 # noinspection PyUnusedLocal
 class BuildOptionParser(object):
     # TODO add nosetests for this class
     platform = None
     bits = None
+    config_file_search_path = [
+        '.', os.path.join(sys.path[0], '..', 'configs'),
+        os.path.join(sys.path[0], '..', '..', 'configs')
+    ]
 
     # add to this list and you can automagically do things like
     # --custom-build-variant-cfg asan
     # and the script will pull up the appropriate path for the config
     # against the current platform and bits.
     # *It will warn and fail if there is not a config for the current
     # platform/bits
     build_variants = {
@@ -513,24 +517,19 @@ class BuildOptionParser(object):
             # this is either an incomplete path or an invalid key in
             # build_variants
             prospective_cfg_path = value
 
         if os.path.exists(prospective_cfg_path):
             # now let's see if we were given a valid pathname
             valid_variant_cfg_path = value
         else:
-            # FIXME: We should actually wait until we have parsed all arguments
-            # before looking at this, otherwise the behavior will depend on the
-            # order of arguments. But that isn't a problem as long as --extra-config-path
-            # is always passed first.
-            config_paths = parser.values.config_paths + [DEFAULT_CONFIG_PATH]
             # let's take our prospective_cfg_path and see if we can
             # determine an existing file
-            for path in config_paths:
+            for path in cls.config_file_search_path:
                 if os.path.exists(os.path.join(path, prospective_cfg_path)):
                     # success! we found a config file
                     valid_variant_cfg_path = os.path.join(path,
                                                           prospective_cfg_path)
                     break
         return valid_variant_cfg_path, prospective_cfg_path
 
     @classmethod