Bug 1401199 - [taskgraph] Use default parameter values when strict=False, r=dustin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 29 Sep 2017 11:35:30 -0400
changeset 383954 a91196fe2eb34d0708b271e285f5d003d2f62c47
parent 383953 e7a7ec82f82de7abbc09a1523ba02865a8b12724
child 383955 f787be169098ff06ff653620cc19196082eff05b
push id32617
push userkwierso@gmail.com
push dateMon, 02 Oct 2017 23:02:55 +0000
treeherdermozilla-central@8c6b4fd1d769 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1401199
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1401199 - [taskgraph] Use default parameter values when strict=False, r=dustin MozReview-Commit-ID: 9XWlLeGcPeQ
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/parameters.py
taskcluster/taskgraph/test/test_parameters.py
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -228,17 +228,17 @@ def get_decision_parameters(options):
         # to run tasks that would otherwise be optimized, ues try_task_config.
         'try_option_syntax': True,
 
         # since no try jobs have been specified, the standard target task will
         # be applied, and tasks should be optimized out of that.
         None: True,
     }[parameters['try_mode']]
 
-    return Parameters(parameters)
+    return Parameters(**parameters)
 
 
 def write_artifact(filename, data):
     logger.info('writing artifact file `{}`'.format(filename))
     if not os.path.isdir(ARTIFACTS_DIR):
         os.mkdir(ARTIFACTS_DIR)
     path = os.path.join(ARTIFACTS_DIR, filename)
     if filename.endswith('.yml'):
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -2,88 +2,115 @@
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
+import time
 import yaml
-from mozbuild.util import ReadOnlyDict
+from datetime import datetime
+
+from mozbuild.util import ReadOnlyDict, memoize
+from mozversioncontrol import get_repository_object
+
+from . import GECKO
 
 
 class ParameterMismatch(Exception):
     """Raised when a parameters.yml has extra or missing parameters."""
 
 
+@memoize
+def get_head_ref():
+    return get_repository_object(GECKO).head_ref
+
+
 # Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
-PARAMETER_NAMES = set([
-    'base_repository',
-    'build_date',
-    'filters',
-    'head_ref',
-    'head_repository',
-    'head_rev',
-    'include_nightly',
-    'level',
-    'message',
-    'moz_build_date',
-    'optimize_target_tasks',
-    'owner',
-    'project',
-    'pushdate',
-    'pushlog_id',
-    'release_history',
-    'target_tasks_method',
-    'try_mode',
-    'try_options',
-    'try_task_config',
-])
+# Parameters are of the form: {name: default}
+PARAMETERS = {
+    'base_repository': 'https://hg.mozilla.org/mozilla-unified',
+    'build_date': lambda: int(time.time()),
+    'filters': ['check_servo', 'target_tasks_method'],
+    'head_ref': get_head_ref,
+    'head_repository': 'https://hg.mozilla.org/mozilla-central',
+    'head_rev': get_head_ref,
+    'include_nightly': False,
+    'level': '3',
+    'message': '',
+    'moz_build_date': lambda: datetime.now().strftime("%Y%m%d%H%M%S"),
+    'optimize_target_tasks': True,
+    'owner': 'nobody@mozilla.com',
+    'project': 'mozilla-central',
+    'pushdate': lambda: int(time.time()),
+    'pushlog_id': '0',
+    'release_history': {},
+    'target_tasks_method': 'default',
+    'try_mode': None,
+    'try_options': None,
+    'try_task_config': None,
+}
 
 
 class Parameters(ReadOnlyDict):
     """An immutable dictionary with nicer KeyError messages on failure"""
+
+    def __init__(self, strict=True, **kwargs):
+        self.strict = strict
+
+        if not self.strict:
+            # apply defaults to missing parameters
+            for name, default in PARAMETERS.items():
+                if name not in kwargs:
+                    if callable(default):
+                        default = default()
+                    kwargs[name] = default
+
+        ReadOnlyDict.__init__(self, **kwargs)
+
     def check(self):
         names = set(self)
+        valid = set(PARAMETERS.keys())
         msg = []
 
-        missing = PARAMETER_NAMES - names
+        missing = valid - names
         if missing:
             msg.append("missing parameters: " + ", ".join(missing))
 
-        extra = names - PARAMETER_NAMES
-        if extra:
+        extra = names - valid
+        if extra and self.strict:
             msg.append("extra parameters: " + ", ".join(extra))
 
         if msg:
             raise ParameterMismatch("; ".join(msg))
 
     def __getitem__(self, k):
-        if k not in PARAMETER_NAMES:
+        if k not in PARAMETERS.keys():
             raise KeyError("no such parameter {!r}".format(k))
         try:
             return super(Parameters, self).__getitem__(k)
         except KeyError:
             raise KeyError("taskgraph parameter {!r} not found".format(k))
 
 
-def load_parameters_file(filename):
+def load_parameters_file(filename, strict=True):
     """
     Load parameters from a path, url, decision task-id or project.
 
     Examples:
         task-id=fdtgsD5DQUmAQZEaGMvQ4Q
         project=mozilla-central
     """
     import urllib
     from taskgraph.util.taskcluster import get_artifact_url, find_task_id
 
     if not filename:
-        return Parameters()
+        return Parameters(strict=strict)
 
     try:
         # reading parameters from a local parameters.yml file
         f = open(filename)
     except IOError:
         # fetching parameters.yml using task task-id, project or supplied url
         task_id = None
         if filename.startswith("task-id="):
@@ -92,13 +119,13 @@ def load_parameters_file(filename):
             index = "gecko.v2.{}.latest.firefox.decision".format(filename.split("=")[1])
             task_id = find_task_id(index)
 
         if task_id:
             filename = get_artifact_url(task_id, 'public/parameters.yml')
         f = urllib.urlopen(filename)
 
     if filename.endswith('.yml'):
-        return Parameters(**yaml.safe_load(f))
+        return Parameters(strict=strict, **yaml.safe_load(f))
     elif filename.endswith('.json'):
-        return Parameters(**json.load(f))
+        return Parameters(strict=strict, **json.load(f))
     else:
         raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
--- a/taskcluster/taskgraph/test/test_parameters.py
+++ b/taskcluster/taskgraph/test/test_parameters.py
@@ -1,23 +1,28 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
-from taskgraph.parameters import Parameters, load_parameters_file, PARAMETER_NAMES
+from taskgraph.parameters import (
+    Parameters,
+    ParameterMismatch,
+    load_parameters_file,
+    PARAMETERS,
+)
 from mozunit import main, MockedOpen
 
 
 class TestParameters(unittest.TestCase):
 
-    vals = {n: n for n in PARAMETER_NAMES}
+    vals = {n: n for n in PARAMETERS.keys()}
 
     def test_Parameters_immutable(self):
         p = Parameters(**self.vals)
 
         def assign():
             p['head_ref'] = 20
         self.assertRaises(Exception, assign)
 
@@ -35,21 +40,27 @@ class TestParameters(unittest.TestCase):
         self.assertEqual(p['head_ref'], 10)
 
     def test_Parameters_check(self):
         p = Parameters(**self.vals)
         p.check()  # should not raise
 
     def test_Parameters_check_missing(self):
         p = Parameters()
-        self.assertRaises(Exception, lambda: p.check())
+        self.assertRaises(ParameterMismatch, lambda: p.check())
+
+        p = Parameters(strict=False)
+        p.check()  # should not raise
 
     def test_Parameters_check_extra(self):
         p = Parameters(xyz=10, **self.vals)
-        self.assertRaises(Exception, lambda: p.check())
+        self.assertRaises(ParameterMismatch, lambda: p.check())
+
+        p = Parameters(strict=False, xyz=10, **self.vals)
+        p.check()  # should not raise
 
     def test_load_parameters_file_yaml(self):
         with MockedOpen({"params.yml": "some: data\n"}):
             self.assertEqual(
                     load_parameters_file('params.yml'),
                     {'some': 'data'})
 
     def test_load_parameters_file_json(self):