Backed out changeset 48687a76aec5 (bug 1535056) for causing nightly builds to fail. a=backout
authorMihai Alexandru Michis <malexandru@mozilla.com>
Tue, 28 May 2019 13:12:23 +0300
changeset 475826 5cc220ddf028de011a922042ee9ba691b94d055d
parent 475823 900ff508219cddc29c6f6307c65f0e90c3422c24
child 475827 fb4e768307c9a11df098b100a42aeb2f6c136802
child 475961 d8321d0b2c5b43933c5e9f201934ec7d4c3a65c0
child 475965 11c950242e5f14dfecac875f352f35bcdd106d00
push id86502
push usermalexandru@mozilla.com
push dateTue, 28 May 2019 10:19:20 +0000
treeherderautoland@fb4e768307c9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1535056
milestone69.0a1
backs out48687a76aec57e2779908d36e2d50817d013504f
first release with
nightly linux32
5cc220ddf028 / 69.0a1 / 20190528101320 / files
nightly linux64
5cc220ddf028 / 69.0a1 / 20190528101320 / files
nightly mac
5cc220ddf028 / 69.0a1 / 20190528101320 / files
nightly win32
5cc220ddf028 / 69.0a1 / 20190528101320 / files
nightly win64
5cc220ddf028 / 69.0a1 / 20190528101320 / files
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
releases
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 48687a76aec5 (bug 1535056) for causing nightly builds to fail. a=backout
taskcluster/taskgraph/parameters.py
taskcluster/taskgraph/test/test_decision.py
taskcluster/taskgraph/test/test_parameters.py
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -3,29 +3,21 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os.path
 import json
+import time
 from datetime import datetime
 
 from mozbuild.util import ReadOnlyDict, memoize
 from mozversioncontrol import get_repository_object
-from taskgraph.util.schema import validate_schema
-from voluptuous import (
-    ALLOW_EXTRA,
-    Any,
-    Inclusive,
-    PREVENT_EXTRA,
-    Required,
-    Schema,
-)
 
 from . import GECKO
 from .util.attributes import release_level
 
 
 class ParameterMismatch(Exception):
     """Raised when a parameters.yml has extra or missing parameters."""
 
@@ -48,146 +40,118 @@ def get_version(product_dir='browser'):
 
 
 def get_app_version(product_dir='browser'):
     app_version_path = os.path.join(GECKO, product_dir, 'config',
                                     'version.txt')
     return get_contents(app_version_path)
 
 
-base_schema = {
-    Required('app_version'): basestring,
-    Required('base_repository'): basestring,
-    Required('build_date'): int,
-    Required('build_number'): int,
-    Inclusive('comm_base_repository', 'comm'): basestring,
-    Inclusive('comm_head_ref', 'comm'): basestring,
-    Inclusive('comm_head_repository', 'comm'): basestring,
-    Inclusive('comm_head_rev', 'comm'): basestring,
-    Required('do_not_optimize'): [basestring],
-    Required('existing_tasks'): {basestring: basestring},
-    Required('filters'): [basestring],
-    Required('head_ref'): basestring,
-    Required('head_repository'): basestring,
-    Required('head_rev'): basestring,
-    Required('hg_branch'): basestring,
-    Required('level'): basestring,
-    Required('message'): basestring,
-    Required('moz_build_date'): basestring,
-    Required('next_version'): Any(None, basestring),
-    Required('optimize_target_tasks'): bool,
-    Required('owner'): basestring,
-    Required('phabricator_diff'): Any(None, basestring),
-    Required('project'): basestring,
-    Required('pushdate'): int,
-    Required('pushlog_id'): basestring,
-    Required('release_enable_emefree'): bool,
-    Required('release_enable_partners'): bool,
-    Required('release_eta'): Any(None, basestring),
-    Required('release_history'): {basestring: [dict]},
-    Required('release_partners'): [basestring],
-    Required('release_partner_config'): Any(None, dict),
-    Required('release_partner_build_number'): int,
-    Required('release_type'): basestring,
-    Required('release_product'): Any(None, basestring),
-    Required('required_signoffs'): [basestring],
-    Required('signoff_urls'): dict,
-    Required('target_tasks_method'): Any(None, basestring),
-    Required('tasks_for'): Any(None, basestring),
-    Required('try_mode'): Any(None, basestring),
-    Required('try_options'): Any(None, dict),
-    Required('try_task_config'): Any(None, dict),
-    Required('version'): basestring,
+# Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
+# Parameters are of the form: {name: default} or {name: lambda: default}
+PARAMETERS = {
+    'app_version': get_app_version(),
+    'base_repository': 'https://hg.mozilla.org/mozilla-unified',
+    'build_date': lambda: int(time.time()),
+    'build_number': 1,
+    'do_not_optimize': [],
+    'existing_tasks': {},
+    'filters': ['target_tasks_method'],
+    'head_ref': get_head_ref,
+    'head_repository': 'https://hg.mozilla.org/mozilla-central',
+    'head_rev': get_head_ref,
+    'hg_branch': 'default',
+    'level': '3',
+    'message': '',
+    'moz_build_date': lambda: datetime.now().strftime("%Y%m%d%H%M%S"),
+    'next_version': None,
+    'optimize_target_tasks': True,
+    'owner': 'nobody@mozilla.com',
+    'project': 'mozilla-central',
+    'pushdate': lambda: int(time.time()),
+    'pushlog_id': '0',
+    'phabricator_diff': None,
+    'release_enable_emefree': False,
+    'release_enable_partners': False,
+    'release_eta': '',
+    'release_history': {},
+    'release_partners': None,
+    'release_partner_config': None,
+    'release_partner_build_number': 1,
+    'release_type': 'nightly',
+    'release_product': None,
+    'required_signoffs': [],
+    'signoff_urls': {},
+    'target_tasks_method': 'default',
+    'tasks_for': 'hg-push',
+    'try_mode': None,
+    'try_options': None,
+    'try_task_config': None,
+    'version': get_version(),
 }
 
-
-COMM_PARAMETERS = [
-    'comm_base_repository',
-    'comm_head_ref',
-    'comm_head_repository',
-    'comm_head_rev',
-]
+COMM_PARAMETERS = {
+    'comm_base_repository': 'https://hg.mozilla.org/comm-central',
+    'comm_head_ref': None,
+    'comm_head_repository': 'https://hg.mozilla.org/comm-central',
+    'comm_head_rev': None,
+}
 
 
 class Parameters(ReadOnlyDict):
     """An immutable dictionary with nicer KeyError messages on failure"""
 
     def __init__(self, strict=True, **kwargs):
         self.strict = strict
 
         if not self.strict:
             # apply defaults to missing parameters
-            kwargs = Parameters._fill_defaults(**kwargs)
+            for name, default in PARAMETERS.items():
+                if name not in kwargs:
+                    if callable(default):
+                        default = default()
+                    kwargs[name] = default
+
+            if set(kwargs) & set(COMM_PARAMETERS.keys()):
+                for name, default in COMM_PARAMETERS.items():
+                    if name not in kwargs:
+                        if callable(default):
+                            default = default()
+                        kwargs[name] = default
 
         ReadOnlyDict.__init__(self, **kwargs)
 
-    @staticmethod
-    def _fill_defaults(**kwargs):
-        now = datetime.utcnow()
-        epoch = datetime.utcfromtimestamp(0)
-        seconds_from_epoch = int((now - epoch).total_seconds())
+    def check(self):
+        names = set(self)
+        valid = set(PARAMETERS.keys())
+        valid_comm = set(COMM_PARAMETERS.keys())
+        msg = []
+
+        missing = valid - names
+        if missing:
+            msg.append("missing parameters: " + ", ".join(missing))
+
+        extra = names - valid
 
-        defaults = {
-            'app_version': get_app_version(),
-            'base_repository': 'https://hg.mozilla.org/mozilla-unified',
-            'build_date': seconds_from_epoch,
-            'build_number': 1,
-            'do_not_optimize': [],
-            'existing_tasks': {},
-            'filters': ['target_tasks_method'],
-            'head_ref': get_head_ref(),
-            'head_repository': 'https://hg.mozilla.org/mozilla-central',
-            'head_rev': get_head_ref(),
-            'hg_branch': 'default',
-            'level': '3',
-            'message': '',
-            'moz_build_date': now.strftime("%Y%m%d%H%M%S"),
-            'next_version': None,
-            'optimize_target_tasks': True,
-            'owner': 'nobody@mozilla.com',
-            'phabricator_diff': None,
-            'project': 'mozilla-central',
-            'pushdate': seconds_from_epoch,
-            'pushlog_id': '0',
-            'release_enable_emefree': False,
-            'release_enable_partners': False,
-            'release_eta': '',
-            'release_history': {},
-            'release_partners': [],
-            'release_partner_config': None,
-            'release_partner_build_number': 1,
-            'release_product': None,
-            'release_type': 'nightly',
-            'required_signoffs': [],
-            'signoff_urls': {},
-            'target_tasks_method': 'default',
-            'tasks_for': 'hg-push',
-            'try_mode': None,
-            'try_options': None,
-            'try_task_config': None,
-            'version': get_version(),
-        }
+        if extra & set(valid_comm):
+            # If any comm_* parameters are specified, ensure all of them are specified.
+            missing = valid_comm - extra
+            if missing:
+                msg.append("missing parameters: " + ", ".join(missing))
+            extra = extra - valid_comm
 
-        if set(COMM_PARAMETERS) & set(kwargs):
-            defaults.update({
-                'comm_base_repository': 'https://hg.mozilla.org/comm-central',
-                'comm_head_repository': 'https://hg.mozilla.org/comm-central',
-            })
+        if extra and self.strict:
+            msg.append("extra parameters: " + ", ".join(extra))
 
-        for name, default in defaults.items():
-            if name not in kwargs:
-                kwargs[name] = default
-
-        return kwargs
-
-    def check(self):
-        schema = Schema(base_schema, extra=PREVENT_EXTRA if self.strict else ALLOW_EXTRA)
-        validate_schema(schema, self.copy(), 'Invalid parameters:')
+        if msg:
+            raise ParameterMismatch("; ".join(msg))
 
     def __getitem__(self, k):
+        if not (k in PARAMETERS.keys() or k in COMM_PARAMETERS.keys()):
+            raise KeyError("no such parameter {!r}".format(k))
         try:
             return super(Parameters, self).__getitem__(k)
         except KeyError:
             raise KeyError("taskgraph parameter {!r} not found".format(k))
 
     def is_try(self):
         """
         Determine whether this graph is being built on a try project or for
--- a/taskcluster/taskgraph/test/test_decision.py
+++ b/taskcluster/taskgraph/test/test_decision.py
@@ -54,62 +54,59 @@ class TestGetDecisionParameters(unittest
     def setUp(self):
         self.options = {
             'base_repository': 'https://hg.mozilla.org/mozilla-unified',
             'head_repository': 'https://hg.mozilla.org/mozilla-central',
             'head_rev': 'abcd',
             'head_ref': 'ef01',
             'message': '',
             'project': 'mozilla-central',
-            'pushlog_id': '143',
+            'pushlog_id': 143,
             'pushdate': 1503691511,
             'owner': 'nobody@mozilla.com',
             'tasks_for': 'hg-push',
-            'level': '3',
+            'level': 3,
         }
 
     @patch('taskgraph.decision.get_hg_revision_branch')
     def test_simple_options(self, mock_get_hg_revision_branch):
         mock_get_hg_revision_branch.return_value = 'default'
         with MockedOpen({self.ttc_file: None}):
             params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
-        self.assertEqual(params['pushlog_id'], '143')
+        self.assertEqual(params['pushlog_id'], 143)
         self.assertEqual(params['build_date'], 1503691511)
         self.assertEqual(params['hg_branch'], 'default')
         self.assertEqual(params['moz_build_date'], '20170825200511')
         self.assertEqual(params['try_mode'], None)
         self.assertEqual(params['try_options'], None)
         self.assertEqual(params['try_task_config'], None)
 
     @patch('taskgraph.decision.get_hg_revision_branch')
-    def test_no_email_owner(self, mock_get_hg_revision_branch):
-        mock_get_hg_revision_branch.return_value = 'default'
+    def test_no_email_owner(self, _):
         self.options['owner'] = 'ffxbld'
         with MockedOpen({self.ttc_file: None}):
             params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
         self.assertEqual(params['owner'], 'ffxbld@noreply.mozilla.org')
 
     @patch('taskgraph.decision.get_hg_revision_branch')
     @patch('taskgraph.decision.get_hg_commit_message')
-    def test_try_options(self, mock_get_hg_commit_message, mock_get_hg_revision_branch):
+    def test_try_options(self, mock_get_hg_commit_message, _):
         mock_get_hg_commit_message.return_value = 'try: -b do -t all'
-        mock_get_hg_revision_branch.return_value = 'default'
         self.options['project'] = 'try'
         with MockedOpen({self.ttc_file: None}):
             params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
         self.assertEqual(params['try_mode'], 'try_option_syntax')
         self.assertEqual(params['try_options']['build_types'], 'do')
         self.assertEqual(params['try_options']['unittests'], 'all')
         self.assertEqual(params['try_task_config'], None)
 
     @patch('taskgraph.decision.get_hg_revision_branch')
     @patch('taskgraph.decision.get_hg_commit_message')
-    def test_try_task_config(self, mock_get_hg_commit_message, mock_get_hg_revision_branch):
+    def test_try_task_config(self, mock_get_hg_commit_message, _):
         mock_get_hg_commit_message.return_value = 'Fuzzy query=foo'
-        mock_get_hg_revision_branch.return_value = 'default'
         ttc = {'tasks': ['a', 'b'], 'templates': {}}
         self.options['project'] = 'try'
         with MockedOpen({self.ttc_file: json.dumps(ttc)}):
             params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
             self.assertEqual(params['try_mode'], 'try_task_config')
             self.assertEqual(params['try_options'], None)
             self.assertEqual(params['try_task_config'], ttc)
 
--- a/taskcluster/taskgraph/test/test_parameters.py
+++ b/taskcluster/taskgraph/test/test_parameters.py
@@ -3,98 +3,62 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
 from taskgraph.parameters import (
     Parameters,
+    ParameterMismatch,
     load_parameters_file,
+    PARAMETERS,
+    COMM_PARAMETERS,
 )
 from mozunit import main, MockedOpen
 
 
 class TestParameters(unittest.TestCase):
 
-    vals = {
-        'app_version': 'app_version',
-        'base_repository': 'base_repository',
-        'build_date': 0,
-        'build_number': 0,
-        'do_not_optimize': [],
-        'existing_tasks': {},
-        'filters': [],
-        'head_ref': 'head_ref',
-        'head_repository': 'head_repository',
-        'head_rev': 'head_rev',
-        'hg_branch': 'hg_branch',
-        'level': 'level',
-        'message': 'message',
-        'moz_build_date': 'moz_build_date',
-        'next_version': 'next_version',
-        'optimize_target_tasks': False,
-        'owner': 'owner',
-        'phabricator_diff': 'phabricator_diff',
-        'project': 'project',
-        'pushdate': 0,
-        'pushlog_id': 'pushlog_id',
-        'release_enable_emefree': False,
-        'release_enable_partners': False,
-        'release_eta': None,
-        'release_history': {},
-        'release_partners': [],
-        'release_partner_config': None,
-        'release_partner_build_number': 1,
-        'release_type': 'release_type',
-        'release_product': None,
-        'required_signoffs': [],
-        'signoff_urls': {},
-        'target_tasks_method': 'target_tasks_method',
-        'tasks_for': 'tasks_for',
-        'try_mode': 'try_mode',
-        'try_options': None,
-        'try_task_config': None,
-        'version': 'version',
-    }
+    vals = {n: n for n in PARAMETERS.keys()}
 
     def test_Parameters_immutable(self):
         p = Parameters(**self.vals)
 
         def assign():
             p['head_ref'] = 20
         self.assertRaises(Exception, assign)
 
     def test_Parameters_missing_KeyError(self):
         p = Parameters(**self.vals)
         self.assertRaises(KeyError, lambda: p['z'])
 
     def test_Parameters_invalid_KeyError(self):
         """even if the value is present, if it's not a valid property, raise KeyError"""
-        p = Parameters(xyz=10, strict=True, **self.vals)
-        self.assertRaises(Exception, lambda: p.check())
+        p = Parameters(xyz=10, **self.vals)
+        self.assertRaises(KeyError, lambda: p['xyz'])
 
     def test_Parameters_get(self):
         p = Parameters(head_ref=10, level=20)
         self.assertEqual(p['head_ref'], 10)
 
     def test_Parameters_check(self):
         p = Parameters(**self.vals)
         p.check()  # should not raise
 
     def test_Parameters_check_missing(self):
         p = Parameters()
-        self.assertRaises(Exception, lambda: p.check())
+        self.assertRaises(ParameterMismatch, lambda: p.check())
 
         p = Parameters(strict=False)
         p.check()  # should not raise
 
     def test_Parameters_check_extra(self):
         p = Parameters(xyz=10, **self.vals)
-        self.assertRaises(Exception, lambda: p.check())
+        self.assertRaises(ParameterMismatch, lambda: p.check())
 
         p = Parameters(strict=False, xyz=10, **self.vals)
         p.check()  # should not raise
 
     def test_load_parameters_file_yaml(self):
         with MockedOpen({"params.yml": "some: data\n"}):
             self.assertEqual(
                     load_parameters_file('params.yml'),
@@ -122,36 +86,31 @@ class TestParameters(unittest.TestCase):
         """
         with MockedOpen({"params.json": '{"some": "data"}'}):
             self.assertEqual(
                 load_parameters_file('params.json', overrides={'some': 'other'}),
                 {'some': 'other'})
 
 
 class TestCommParameters(unittest.TestCase):
-    vals = dict({
-        'comm_base_repository': 'comm_base_repository',
-        'comm_head_ref': 'comm_head_ref',
-        'comm_head_repository': 'comm_head_repository',
-        'comm_head_rev': 'comm_head_rev',
-    }.items() + TestParameters.vals.items())
+    vals = {n: n for n in PARAMETERS.keys() + COMM_PARAMETERS.keys()}
 
     def test_Parameters_check(self):
         """
         Specifying all of the gecko and comm parameters doesn't result in an error.
         """
         p = Parameters(**self.vals)
         p.check()  # should not raise
 
     def test_Parameters_check_missing(self):
         """
         If any of the comm parameters are specified, all of them must be specified.
         """
         vals = self.vals.copy()
-        del vals['comm_base_repository']
+        del vals[next(iter(COMM_PARAMETERS.keys()))]
         p = Parameters(**vals)
         self.assertRaises(Exception, p.check)
 
     def test_Parameters_check_extra(self):
         """
         If parameters other than the global and comm parameters are specified,
         an error is reported.
         """