Bug 1539905 Cache yaml loading with copy of memoized values r=mtabara a=release
authorSimon Fraser <sfraser@mozilla.com>
Fri, 05 Apr 2019 20:36:37 +0300
changeset 525959 864e16abd2f73a5a0d0ef7a887a3a90e0ee22a37
parent 525958 df05571b936fe6b87dae764994759adabe383166
child 525960 3dc7b1a572313386a459b174e7335847e3c1dbb2
push id2032
push userffxbld-merge
push dateMon, 13 May 2019 09:36:57 +0000
treeherdermozilla-release@455c1065dcbe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara, release
bugs1539905
milestone67.0
Bug 1539905 Cache yaml loading with copy of memoized values r=mtabara a=release
taskcluster/taskgraph/util/scriptworker.py
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -17,18 +17,21 @@ Additional configuration is found in the
 """
 from __future__ import absolute_import, print_function, unicode_literals
 import functools
 import json
 import os
 import itertools
 from copy import deepcopy
 from datetime import datetime
+
 import jsone
 
+from mozbuild.util import memoize
+
 from .schema import resolve_keyed_by
 from .taskcluster import get_artifact_prefix
 from .yaml import load_yaml
 
 # constants {{{1
 """Map signing scope aliases to sets of projects.
 
 Currently m-c and DevEdition on m-b use nightly signing; Beta on m-b and m-r
@@ -320,16 +323,18 @@ get_balrog_server_scope = functools.part
 )
 
 get_push_apk_scope = functools.partial(
     get_scope_from_project,
     alias_to_project_map=PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
     alias_to_scope_map=PUSH_APK_SCOPES,
 )
 
+cached_load_yaml = memoize(load_yaml)
+
 
 # release_config {{{1
 def get_release_config(config):
     """Get the build number and version for a release task.
 
     Currently only applies to beetmover tasks.
 
     Args:
@@ -417,17 +422,17 @@ def generate_beetmover_upstream_artifact
     """
     base_artifact_prefix = get_artifact_prefix(job)
     resolve_keyed_by(
         job, 'attributes.artifact_map',
         'artifact map',
         project=config.params['project'],
         platform=platform
     )
-    map_config = load_yaml(job['attributes']['artifact_map'])
+    map_config = deepcopy(cached_load_yaml(job['attributes']['artifact_map']))
     upstream_artifacts = list()
 
     if not locale:
         locales = map_config['default_locales']
     elif isinstance(locale, list):
         locales = locale
     else:
         locales = [locale]
@@ -489,17 +494,17 @@ def generate_beetmover_compressed_upstre
     Args:
         job (dict): The current job being generated
         dependencies (list): A list of the job's dependency labels.
 
     Returns:
         list: A list of dictionaries conforming to the upstream_artifacts spec.
     """
     base_artifact_prefix = get_artifact_prefix(job)
-    map_config = load_yaml(job['attributes']['artifact_map'])
+    map_config = deepcopy(cached_load_yaml(job['attributes']['artifact_map']))
     upstream_artifacts = list()
 
     if not dependencies:
         dependencies = job['dependencies'].keys()
 
     for dep in dependencies:
         paths = list()
 
@@ -546,17 +551,17 @@ def generate_beetmover_artifact_map(conf
     """
     platform = kwargs.get('platform', '')
     resolve_keyed_by(
         job, 'attributes.artifact_map',
         'artifact map',
         project=config.params['project'],
         platform=platform
     )
-    map_config = load_yaml(job['attributes']['artifact_map'])
+    map_config = deepcopy(cached_load_yaml(job['attributes']['artifact_map']))
     base_artifact_prefix = map_config.get('base_artifact_prefix', get_artifact_prefix(job))
 
     artifacts = list()
 
     dependencies = job['dependencies'].keys()
 
     if kwargs.get('locale'):
         if isinstance(kwargs['locale'], list):
@@ -689,17 +694,17 @@ def generate_beetmover_partials_artifact
     """
     platform = kwargs.get('platform', '')
     resolve_keyed_by(
         job, 'attributes.artifact_map',
         'artifact map',
         project=config.params['project'],
         platform=platform
     )
-    map_config = load_yaml(job['attributes']['artifact_map'])
+    map_config = deepcopy(cached_load_yaml(job['attributes']['artifact_map']))
     base_artifact_prefix = map_config.get('base_artifact_prefix', get_artifact_prefix(job))
 
     artifacts = list()
     dependencies = job['dependencies'].keys()
 
     if kwargs.get('locale'):
         locales = [kwargs['locale']]
     else: