Turn the release_history file into part of the parameters, and filter on nightly as best we can DONTBUILD
authorSimon Fraser <sfraser@mozilla.com>
Thu, 07 Sep 2017 17:45:54 +0100
changeset 663928 0cb73421f0501797172943f4177c4013b9bae573
parent 663927 a644680025fbd12008ced08241332d2b50eba98a
child 663929 cdb505cb5f4c924874224259050d14529f99afcc
push id79558
push usercatlee@mozilla.com
push dateWed, 13 Sep 2017 13:53:28 +0000
milestone57.0a1
Turn the release_history file into part of the parameters, and filter on nightly as best we can DONTBUILD
taskcluster/docs/parameters.rst
taskcluster/mach_commands.py
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/util/partials.py
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -103,19 +103,20 @@ syntax or reading a project-specific con
 
 ``optimize_target_tasks``
     If true, then target tasks are eligible for optimization.
 
 ``include_nightly``
     If true, then nightly tasks are eligible for optimization.
 
 ``release_history``
-   A path to a filename containing the releases history for this project
-   from balrog. Used by the partial update generation. If missing, no
-   partials will be generated.
+   History of recent releases by platform and locale, used when generating
+   partial updates for nightly releases.
+   Suitable contents can be generated with ``mach release-history``,
+   which will print to the console by default.
 
 Morphed Set
 -----------
 
 ``morph_templates``
     Dict of JSON-e templates to apply to each task, keyed by template name.
     Values are extra context that will be available to the template under the
     ``input.<template>`` key. Available templates live in
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -502,23 +502,24 @@ class TaskClusterImagesProvider(object):
             else:
                 build_context(image_name, context_only)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
 
 @CommandProvider
 class TaskClusterPartialsData(object):
-    @Command('partials', category="ci",
-             description="Qeury balrog for build history to enable partials")
+    @Command('release-history', category="ci",
+             description="Query balrog for release history used by enable partials generation")
     @CommandArgument('-b', '--branch',
-                     help="The project branch used in balrog branch, such as "
+                     help="The project branch used in balrog, such as "
                           "mozilla-central, release, date")
     @CommandArgument('--product', default='Firefox',
                      help="The product identifier, such as 'Firefox'")
     def generate_partials_builds(self, product, branch):
         from taskgraph.util.partials import populate_release_history
         try:
-            build_history = populate_release_history(product, branch)
-            print(json.dumps(build_history, sort_keys=True, indent=2, separators=(',', ': ')))
+            import yaml
+            release_history = {'release_history': populate_release_history(product, branch)}
+            print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
         except Exception:
             traceback.print_exc()
             sys.exit(1)
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -104,21 +104,16 @@ def taskgraph_decision(options):
      * running task-graph generation exactly the same way the other `mach
        taskgraph` commands do
      * generating a set of artifacts to memorialize the graph
      * calling TaskCluster APIs to create the graph
     """
 
     parameters = get_decision_parameters(options)
 
-    # Release history for partials generation, nightly only.
-    release_history = populate_release_history('Firefox', parameters['project'])
-
-    write_artifact(parameters['release_history'], release_history)
-
     # create a TaskGraphGenerator instance
     tgg = TaskGraphGenerator(
         root_dir=options['root'],
         parameters=parameters)
 
     # write out the parameters used to generate this graph
     write_artifact('parameters.yml', dict(**parameters))
 
@@ -171,20 +166,16 @@ def get_decision_parameters(options):
     # custom filters.
     parameters['filters'] = [
         'check_servo',
         'target_tasks_method',
     ]
     parameters['target_task_labels'] = []
     parameters['morph_templates'] = {}
 
-    # Define default build history file, to store balrog data required by
-    # partials generation
-    parameters['release_history'] = 'release_history.json'
-
     # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
     # case, fake it
     if '@' not in parameters['owner']:
         parameters['owner'] += '@noreply.mozilla.org'
 
     # use the pushdate as build_date if given, else use current time
     parameters['build_date'] = parameters['pushdate'] or int(time.time())
     # moz_build_date is the build identifier based on build_date
@@ -208,16 +199,20 @@ def get_decision_parameters(options):
             task_config = json.load(fh)
         parameters['morph_templates'] = task_config.get('templates', {})
         parameters['target_task_labels'] = task_config.get('tasks')
 
     # `target_tasks_method` has higher precedence than `project` parameters
     if options.get('target_tasks_method'):
         parameters['target_tasks_method'] = options['target_tasks_method']
 
+    parameters.setdefault('release_history', dict())
+    if 'nightly' in parameters.get('target_tasks_method', ''):
+        parameters['release_history'] = populate_release_history('Firefox', project)
+
     return Parameters(parameters)
 
 
 def write_artifact(filename, data):
     logger.info('writing artifact file `{}`'.format(filename))
     if not os.path.isdir(ARTIFACTS_DIR):
         os.mkdir(ARTIFACTS_DIR)
     path = os.path.join(ARTIFACTS_DIR, filename)
--- a/taskcluster/taskgraph/util/partials.py
+++ b/taskcluster/taskgraph/util/partials.py
@@ -49,57 +49,43 @@ def get_friendly_platform_name(platform)
     """Convert build platform names into friendly platform names"""
     if '-nightly' in platform:
         platform = platform.replace('-nightly', '')
     if '-devedition' in platform:
         platform = platform.replace('-devedition', '')
     return PLATFORM_RENAMES.get(platform, platform)
 
 
-def _open_release_history(release_history):
-    # TODO the join here is fragile. Need to reliably determine artifact path
-    try:
-        with open(os.path.join('artifacts', release_history), 'r') as f:
-            return json.load(f)
-    except IOError:
-        return dict()
-    return dict()
-
-
 def _sanitize_platform(platform):
     platform = get_friendly_platform_name(platform)
     if platform not in BALROG_PLATFORM_MAP:
         return platform
     return BALROG_PLATFORM_MAP[platform][0]
 
 
 def get_builds(release_history, platform, locale):
     """Examine cached balrog release history and return the list of
     builds we need to generate diffs from"""
-    history = _open_release_history(release_history)
     platform = _sanitize_platform(platform)
-    return history.get(platform, {}).get(locale, {})
+    return release_history.get(platform, {}).get(locale, {})
 
 
 def get_partials_artifacts(release_history, platform, locale):
-    history = _open_release_history(release_history)
     platform = _sanitize_platform(platform)
-    return history.get(platform, {}).get(locale, {}).keys()
+    return release_history.get(platform, {}).get(locale, {}).keys()
 
 
 def get_partials_artifact_map(release_history, platform, locale):
-    history = _open_release_history(release_history)
     platform = _sanitize_platform(platform)
-    return {k: history[platform][locale][k]['buildid'] for k in history.get(platform, {}).get(locale, {})}
+    return {k: release_history[platform][locale][k]['buildid'] for k in release_history.get(platform, {}).get(locale, {})}
 
 
 def get_partials_artifacts_friendly(release_history, platform, locale, current_builid):
-    history = _open_release_history(release_history)
     platform = _sanitize_platform(platform)
-    return history.get(platform, {}).get(locale, {}).keys()
+    return release_history.get(platform, {}).get(locale, {}).keys()
 
 
 def _retry_on_http_errors(url, verify, params, errors):
     for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10):
         try:
             req = requests.get(url, verify=verify, params=params)
             req.raise_for_status()
             return req