Bug 1456234: [mozharness] Process EXTRA_MOZHARNESS_CONFIG even if no files are specified; r=aki
authorTom Prince <mozilla@hocat.ca>
Mon, 23 Apr 2018 17:39:35 -0600
changeset 468729 3dba61b12c7d1478558550257125d36f8b925fcc
parent 468728 d759ec86eb3832bced3376dbdf31f6da5bd052e8
child 468730 5f006fc7deaf4837a247a81f0158aebba7f5ca57
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1456234
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1456234: [mozharness] Process EXTRA_MOZHARNESS_CONFIG even if no files are specified; r=aki This will allow mozharness configs to be specified exclusively in the taskgraph. Differential Revision: https://phabricator.services.mozilla.com/D1017
taskcluster/scripts/builder/build-linux.sh
testing/mozharness/mozharness/base/config.py
--- a/taskcluster/scripts/builder/build-linux.sh
+++ b/taskcluster/scripts/builder/build-linux.sh
@@ -48,17 +48,17 @@ export LIBRARY_PATH=$LIBRARY_PATH:$WORKS
 
 if [[ -n ${USE_SCCACHE} ]]; then
     # Point sccache at the Taskcluster proxy for AWS credentials.
     export AWS_IAM_CREDENTIALS_URL="http://taskcluster/auth/v1/aws/s3/read-write/taskcluster-level-${MOZ_SCM_LEVEL}-sccache-${TASKCLUSTER_WORKER_GROUP}/?format=iam-role-compat"
 fi
 
 # test required parameters are supplied
 if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
-if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi
+if [[ -z "${MOZHARNESS_CONFIG}" && -z "${EXTRA_MOZHARNESS_CONFIG}" ]]; then fail "MOZHARNESS_CONFIG or EXTRA_MOZHARNESS_CONFIG is not set"; fi
 
 # run XVfb in the background, if necessary
 if $NEED_XVFB; then
     . /builds/worker/scripts/xvfb.sh
 
     cleanup() {
         local rv=$?
         cleanup_xvfb
--- a/testing/mozharness/mozharness/base/config.py
+++ b/testing/mozharness/mozharness/base/config.py
@@ -280,18 +280,19 @@ class BaseConfig(object):
             type="string", default=os.getcwd(),
             help="Specify the absolute path of the parent of the working directory"
         )
         self.config_parser.add_option(
             "--extra-config-path", action='extend', dest="config_paths",
             type="string", help="Specify additional paths to search for config files.",
         )
         self.config_parser.add_option(
-            "-c", "--config-file", "--cfg", action="extend", dest="config_files",
-            type="string", help="Specify a config file; can be repeated"
+            "-c", "--config-file", "--cfg", action="extend",
+            dest="config_files", default=[], type="string",
+            help="Specify a config file; can be repeated",
         )
         self.config_parser.add_option(
             "-C", "--opt-config-file", "--opt-cfg", action="extend",
             dest="opt_config_files", type="string", default=[],
             help="Specify an optional config file, like --config-file but with no "
                  "error if the file is missing; can be repeated"
         )
         self.config_parser.add_option(
@@ -484,45 +485,46 @@ class BaseConfig(object):
         defaults = self.config_parser.defaults.copy()
 
         if not options.config_files:
             if self.require_config_file:
                 if options.list_actions:
                     self.list_actions()
                 print("Required config file not set! (use --config-file option)")
                 raise SystemExit(-1)
+
+        # this is what get_cfgs_from_files returns. It will represent each
+        # config file name and its assoctiated dict
+        # eg ('builds/branch_specifics.py', {'foo': 'bar'})
+        # let's store this to self for things like --interpret-config-files
+        self.all_cfg_files_and_dicts.extend(self.get_cfgs_from_files(
+            # append opt_config to allow them to overwrite previous configs
+            options.config_files + options.opt_config_files, options=options
+        ))
+        config = {}
+        if (self.append_env_variables_from_configs
+                or options.append_env_variables_from_configs):
+            # We only append values from various configs for the 'env' entry
+            # For everything else we follow the standard behaviour
+            for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
+                for v in c_dict.keys():
+                    if v == 'env' and v in config:
+                        config[v].update(c_dict[v])
+                    else:
+                        config[v] = c_dict[v]
         else:
-            # this is what get_cfgs_from_files returns. It will represent each
-            # config file name and its assoctiated dict
-            # eg ('builds/branch_specifics.py', {'foo': 'bar'})
-            # let's store this to self for things like --interpret-config-files
-            self.all_cfg_files_and_dicts.extend(self.get_cfgs_from_files(
-                # append opt_config to allow them to overwrite previous configs
-                options.config_files + options.opt_config_files, options=options
-            ))
-            config = {}
-            if (self.append_env_variables_from_configs
-                    or options.append_env_variables_from_configs):
-                # We only append values from various configs for the 'env' entry
-                # For everything else we follow the standard behaviour
-                for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
-                    for v in c_dict.keys():
-                        if v == 'env' and v in config:
-                            config[v].update(c_dict[v])
-                        else:
-                            config[v] = c_dict[v]
-            else:
-                for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
-                    config.update(c_dict)
-            # assign or update self._config depending on if it exists or not
-            #    NOTE self._config will be passed to ReadOnlyConfig's init -- a
-            #    dict subclass with immutable locking capabilities -- and serve
-            #    as the keys/values that make up that instance. Ultimately,
-            #    this becomes self.config during BaseScript's init
-            self.set_config(config)
+            for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
+                config.update(c_dict)
+        # assign or update self._config depending on if it exists or not
+        #    NOTE self._config will be passed to ReadOnlyConfig's init -- a
+        #    dict subclass with immutable locking capabilities -- and serve
+        #    as the keys/values that make up that instance. Ultimately,
+        #    this becomes self.config during BaseScript's init
+        self.set_config(config)
+
         for key in defaults.keys():
             value = getattr(options, key)
             if value is None:
                 continue
             # Don't override config_file defaults with config_parser defaults
             if key in defaults and value == defaults[key] and key in self._config:
                 continue
             self._config[key] = value