bug 1423081 - add firefox + devedition relpro. r=callek
authorAki Sasaki <asasaki@mozilla.com>
Tue, 05 Dec 2017 19:09:19 -0800
changeset 448078 77de103de179a8169f4d2609280357a63eed44a2
parent 448077 89cf53d0d832f68ad969e07bb8be6365b46ef9cb
child 448079 35d738241395d6f4b182151f12ea71313e904900
push id8527
push userCallek@gmail.com
push dateThu, 11 Jan 2018 21:05:50 +0000
treeherdermozilla-beta@95342d212a7a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscallek
bugs1423081
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1423081 - add firefox + devedition relpro. r=callek Add support for the three firefox and devedition relpro flavors (we could probably reduce devedition to 2). Also, instead of defining which kinds to use from the previous graph in `previous_graph_kinds`, specify which kinds to rebuild (ignore) from the previous graph in `rebuild_kinds`. This list will be much smaller (currently empty). MozReview-Commit-ID: 5rH1TW7GbAD
taskcluster/docs/parameters.rst
taskcluster/taskgraph/actions/release_promotion.py
taskcluster/taskgraph/actions/util.py
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/parameters.py
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -142,16 +142,20 @@ Release Promotion
 -----------------
 
 ``build_number``
    Specify the release promotion build number.
 
 ``next_version``
    Specify the next version for version bump tasks.
 
+``desktop_release_type``
+   The type of desktop release being promoted. One of "beta", "devedition", "esr", "rc",
+   or "release".
+
 Comm Push Information
 ---------------------
 
 These parameters correspond to the repository and revision of the comm-central
 repository to checkout. Their meaning is the same as the corresponding
 parameters for the gecko repository above. They are optional, but if any of
 them are specified, they must all be specified.
 
--- a/taskcluster/taskgraph/actions/release_promotion.py
+++ b/taskcluster/taskgraph/actions/release_promotion.py
@@ -1,69 +1,89 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import json
+import os
+
 from .registry import register_callback_action
 
 from .util import (find_decision_task, find_existing_tasks_from_previous_kinds,
                    find_hg_revision_pushlog_id)
 from taskgraph.util.taskcluster import get_artifact
+from taskgraph.util.partials import populate_release_history
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.decision import taskgraph_decision
 from taskgraph.parameters import Parameters
 from taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS
 
 RELEASE_PROMOTION_CONFIG = {
     'promote_fennec': {
-        'target_tasks_method': 'candidates_fennec',
-        'previous_graph_kinds': [
-            'build', 'build-signing', 'repackage', 'repackage-signing',
-            "beetmover", "beetmover-checksums", "checksums-signing",
-            "nightly-l10n", "nightly-l10n-signing", "release-bouncer-sub",
-            "upload-generated-sources", "upload-symbols",
-        ],
-        'do_not_optimize': [],
+        'target_tasks_method': 'promote_fennec',
     },
-    'publish_fennec': {
-        'target_tasks_method': 'publish_fennec',
-        'previous_graph_kinds': [
-            'build', 'build-signing', 'repackage', 'repackage-signing',
-            'release-bouncer-sub', 'beetmover', 'beetmover-checksums',
-            'beetmover-l10n', 'beetmover-repackage',
-            'beetmover-repackage-signing', "checksums-signing",
-            'release-notify-promote',
-        ],
-        'do_not_optimize': [],
+    'ship_fennec': {
+        'target_tasks_method': 'ship_fennec',
     },
     'promote_firefox': {
-        'target_tasks_method': '{project}_desktop_promotion',
-        'previous_graph_kinds': [
-            'build', 'build-signing', 'repackage', 'repackage-signing',
-        ],
-        'do_not_optimize': [],
+        'target_tasks_method': 'promote_firefox',
+    },
+    'push_firefox': {
+        'target_tasks_method': 'push_firefox',
+    },
+    'ship_firefox': {
+        'target_tasks_method': 'ship_firefox',
     },
-    'publish_firefox': {
-        'target_tasks_method': 'publish_firefox',
-        'previous_graph_kinds': [
-            'build', 'build-signing', 'repackage', 'repackage-signing',
-            'nightly-l10n', 'nightly-l10n-signing', 'repackage-l10n',
-        ],
-        'do_not_optimize': [],
+    'promote_devedition': {
+        'target_tasks_method': 'promote_devedition',
+    },
+    'push_devedition': {
+        'target_tasks_method': 'push_devedition',
+    },
+    'ship_devedition': {
+        'target_tasks_method': 'ship_devedition',
     },
 }
 
 VERSION_BUMP_FLAVORS = (
-    'publish_fennec',
-    'publish_firefox',
-    'publish_devedition',
+    'ship_fennec',
+    'ship_firefox',
+    'ship_devedition',
+)
+
+UPTAKE_MONITORING_PLATFORMS_FLAVORS = (
+    'push_firefox',
+    'push_devedition',
+)
+
+PARTIAL_UPDATES_FLAVORS = UPTAKE_MONITORING_PLATFORMS_FLAVORS + (
+    'promote_firefox',
+    'promote_devedition',
+)
+
+DESKTOP_RELEASE_TYPE_FLAVORS = (
+    'promote_firefox',
+    'push_firefox',
+    'ship_firefox',
+    'promote_devedition',
+    'push_devedition',
+    'ship_devedition',
+)
+
+
+VALID_DESKTOP_RELEASE_TYPES = (
+    'beta',
+    'devedition',
+    'esr',
+    'release',
+    'rc',
 )
 
 
 def is_release_promotion_available(parameters):
     return parameters['project'] in RELEASE_PROMOTION_PROJECTS
 
 
 @register_callback_action(
@@ -108,19 +128,19 @@ def is_release_promotion_available(param
                 'enum': sorted(RELEASE_PROMOTION_CONFIG.keys()),
             },
             'target_tasks_method': {
                 'type': 'string',
                 'title': 'target task method',
                 'description': ('Optional: the target task method to use to generate '
                                 'the new graph.'),
             },
-            'previous_graph_kinds': {
+            'rebuild_kinds': {
                 'type': 'array',
-                'description': ('Optional: an array of kinds to use from the previous '
+                'description': ('Optional: an array of kinds to ignore from the previous '
                                 'graph(s).'),
                 'items': {
                     'type': 'string',
                 },
             },
             'previous_graph_ids': {
                 'type': 'array',
                 'description': ('Optional: an array of taskIds of decision or action '
@@ -130,62 +150,159 @@ def is_release_promotion_available(param
                     'type': 'string',
                 },
             },
             'next_version': {
                 'type': 'string',
                 'description': 'Next version.',
                 'default': '',
             },
+
+            # Example:
+            #   'partial_updates': {
+            #       '38.0': {
+            #           'buildNumber': 1,
+            #           'locales': ['de', 'en-GB', 'ru', 'uk', 'zh-TW']
+            #       },
+            #       '37.0': {
+            #           'buildNumber': 2,
+            #           'locales': ['de', 'en-GB', 'ru', 'uk']
+            #       }
+            #   }
+            'partial_updates': {
+                'type': 'object',
+                'description': 'Partial updates.',
+                'default': {},
+                'additionalProperties': {
+                    'type': 'object',
+                    'properties': {
+                        'buildNumber': {
+                            'type': 'number',
+                        },
+                        'locales': {
+                            'type': 'array',
+                            'items':  {
+                                'type': 'string',
+                            },
+                        },
+                    },
+                    'required': [
+                        'buildNumber',
+                        'locales',
+                    ],
+                    'additionalProperties': False,
+                }
+            },
+
+            'uptake_monitoring_platforms': {
+                'type': 'array',
+                'items': {
+                    'type': 'string',
+                    'enum': [
+                        'macosx',
+                        'win32',
+                        'win64',
+                        'linux',
+                        'linux64',
+                    ],
+                },
+                'default': [],
+            },
+
+            'desktop_release_type': {
+                'type': 'string',
+                'default': '',
+            },
         },
         "required": ['release_promotion_flavor', 'build_number'],
     }
 )
 def release_promotion_action(parameters, input, task_group_id, task_id, task):
     release_promotion_flavor = input['release_promotion_flavor']
+    release_history = {}
+    desktop_release_type = None
+
     next_version = str(input.get('next_version') or '')
     if release_promotion_flavor in VERSION_BUMP_FLAVORS:
         # We force str() the input, hence the 'None'
         if next_version in ['', 'None']:
             raise Exception(
                 "`next_version` property needs to be provided for %s "
                 "targets." % ', '.join(VERSION_BUMP_FLAVORS)
             )
+
+    if release_promotion_flavor in PARTIAL_UPDATES_FLAVORS:
+        partial_updates = json.dumps(input.get('partial_updates', {}))
+        if partial_updates == "{}":
+            raise Exception(
+                "`partial_updates` property needs to be provided for %s "
+                "targets." % ', '.join(PARTIAL_UPDATES_FLAVORS)
+            )
+        os.environ['PARTIAL_UPDATES'] = partial_updates
+        release_history = populate_release_history(
+            'Firefox', parameters['project'], partial_updates=input['partial_updates']
+        )
+
+    if release_promotion_flavor in UPTAKE_MONITORING_PLATFORMS_FLAVORS:
+        uptake_monitoring_platforms = json.dumps(input.get('uptake_monitoring_platforms', []))
+        if partial_updates == "[]":
+            raise Exception(
+                "`uptake_monitoring_platforms` property needs to be provided for %s "
+                "targets." % ', '.join(UPTAKE_MONITORING_PLATFORMS_FLAVORS)
+            )
+        os.environ['UPTAKE_MONITORING_PLATFORMS'] = uptake_monitoring_platforms
+
+    if release_promotion_flavor in DESKTOP_RELEASE_TYPE_FLAVORS:
+        desktop_release_type = input.get('desktop_release_type', None)
+        if desktop_release_type not in VALID_DESKTOP_RELEASE_TYPES:
+            raise Exception("`desktop_release_type` must be one of: %s" %
+                            ", ".join(VALID_DESKTOP_RELEASE_TYPES))
+
     promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor]
 
     target_tasks_method = input.get(
         'target_tasks_method',
         promotion_config['target_tasks_method'].format(project=parameters['project'])
     )
-    previous_graph_kinds = input.get(
-        'previous_graph_kinds', promotion_config['previous_graph_kinds']
+    rebuild_kinds = input.get(
+        'rebuild_kinds', promotion_config.get('rebuild_kinds', [])
     )
     do_not_optimize = input.get(
-        'do_not_optimize', promotion_config['do_not_optimize']
+        'do_not_optimize', promotion_config.get('do_not_optimize', [])
     )
 
     # make parameters read-write
     parameters = dict(parameters)
     # Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
     # or ``revision``.
     previous_graph_ids = input.get('previous_graph_ids')
     if not previous_graph_ids:
         revision = input.get('revision')
         parameters['pushlog_id'] = parameters['pushlog_id'] or \
             find_hg_revision_pushlog_id(parameters, revision)
         previous_graph_ids = [find_decision_task(parameters)]
 
-    # Download parameters and full task graph from the first decision task.
+    # Download parameters from the first decision task
     parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
-    full_task_graph = get_artifact(previous_graph_ids[0], "public/full-task-graph.json")
-    _, full_task_graph = TaskGraph.from_json(full_task_graph)
+    # Download and combine full task graphs from each of the previous_graph_ids.
+    # Sometimes previous relpro action tasks will add tasks, like partials,
+    # that didn't exist in the first full_task_graph, so combining them is
+    # important. The rightmost graph should take precedence in the case of
+    # conflicts.
+    combined_full_task_graph = {}
+    for graph_id in previous_graph_ids:
+        full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
+        combined_full_task_graph.update(full_task_graph)
+    _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
     parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
-        full_task_graph, previous_graph_ids, previous_graph_kinds
+        combined_full_task_graph, previous_graph_ids, rebuild_kinds
     )
     parameters['do_not_optimize'] = do_not_optimize
     parameters['target_tasks_method'] = target_tasks_method
-    parameters['build_number'] = str(input['build_number'])
+    parameters['build_number'] = int(input['build_number'])
     parameters['next_version'] = next_version
+    parameters['release_history'] = release_history
+    parameters['desktop_release_type'] = desktop_release_type
 
     # make parameters read-only
     parameters = Parameters(**parameters)
 
     taskgraph_decision({}, parameters=parameters)
--- a/taskcluster/taskgraph/actions/util.py
+++ b/taskcluster/taskgraph/actions/util.py
@@ -43,25 +43,25 @@ def find_hg_revision_pushlog_id(paramete
             "Unable to find a single pushlog_id for {} revision {}: {}".format(
                 parameters['head_repository'], revision, pushes
             )
         )
     return pushes[0]
 
 
 def find_existing_tasks_from_previous_kinds(full_task_graph, previous_graph_ids,
-                                            previous_graph_kinds):
-    """Given a list of previous decision/action taskIds and kinds to replace
+                                            rebuild_kinds):
+    """Given a list of previous decision/action taskIds and kinds to ignore
     from the previous graphs, return a dictionary of labels-to-taskids to use
     as ``existing_tasks`` in the optimization step."""
     existing_tasks = {}
     for previous_graph_id in previous_graph_ids:
         label_to_taskid = get_artifact(previous_graph_id, "public/label-to-taskid.json")
         kind_labels = set(t.label for t in full_task_graph.tasks.itervalues()
-                          if t.attributes['kind'] in previous_graph_kinds)
+                          if t.attributes['kind'] not in rebuild_kinds)
         for label in set(label_to_taskid.keys()).intersection(kind_labels):
             existing_tasks[label] = label_to_taskid[label]
     return existing_tasks
 
 
 def fetch_graph_and_labels(parameters):
     decision_task_id = find_decision_task(parameters)
 
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -189,16 +189,17 @@ def get_decision_parameters(options):
     parameters['filters'] = [
         'check_servo',
         'target_tasks_method',
     ]
     parameters['existing_tasks'] = {}
     parameters['do_not_optimize'] = []
     parameters['build_number'] = 1
     parameters['next_version'] = None
+    parameters['desktop_release_type'] = ''
 
     # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
     # case, fake it
     if '@' not in parameters['owner']:
         parameters['owner'] += '@noreply.mozilla.org'
 
     # use the pushdate as build_date if given, else use current time
     parameters['build_date'] = parameters['pushdate'] or int(time.time())
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -27,16 +27,17 @@ def get_head_ref():
 
 
 # Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
 # Parameters are of the form: {name: default}
 PARAMETERS = {
     'base_repository': 'https://hg.mozilla.org/mozilla-unified',
     'build_date': lambda: int(time.time()),
     'build_number': 1,
+    'desktop_release_type': '',
     'do_not_optimize': [],
     'existing_tasks': {},
     'filters': ['check_servo', 'target_tasks_method'],
     'head_ref': get_head_ref,
     'head_repository': 'https://hg.mozilla.org/mozilla-central',
     'head_rev': get_head_ref,
     'include_nightly': False,
     'level': '3',