Bug 1454685: [taskgraph] Load docker images based on digest generated by the task; r=dustin
authorTom Prince <mozilla@hocat.ca>
Wed, 28 Nov 2018 18:18:04 +0000
changeset 504989 d943368c547a12ec273f0d71b21ddc90e1ff3062
parent 504988 eefaa80b67451e8213c63421a7cbccc2824513dd
child 504990 94b6dac6695dff0236560b2ee92ea0fb749e2e0e
push id10290
push userffxbld-merge
push dateMon, 03 Dec 2018 16:23:23 +0000
treeherdermozilla-beta@700bed2445e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1454685
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1454685: [taskgraph] Load docker images based on digest generated by the task; r=dustin Differential Revision: https://phabricator.services.mozilla.com/D12167
taskcluster/taskgraph/docker.py
taskcluster/taskgraph/util/cached_tasks.py
--- a/taskcluster/taskgraph/docker.py
+++ b/taskcluster/taskgraph/docker.py
@@ -6,38 +6,45 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import os
 import tarfile
 from io import BytesIO
 
+from taskgraph.parameters import Parameters
+from taskgraph.optimize import IndexSearch
 from taskgraph.util import docker
 from taskgraph.util.taskcluster import (
-    find_task_id,
     get_artifact_url,
     get_session,
 )
-from taskgraph.util.cached_tasks import cached_index_path
+from taskgraph.generator import load_tasks_for_kind
 from . import GECKO
 
 
 def load_image_by_name(image_name, tag=None):
-    context_path = os.path.join(GECKO, 'taskcluster', 'docker', image_name)
-    context_hash = docker.generate_context_hash(GECKO, context_path, image_name)
+    params = Parameters(
+        level=os.environ.get('MOZ_SCM_LEVEL', '3'),
+        strict=False,
+    )
+    tasks = load_tasks_for_kind(params, 'docker-image')
+    task = tasks['build-docker-image-{}'.format(image_name)]
+    task_id = IndexSearch().should_replace_task(
+        task, {}, task.optimization.get('index-search', []))
 
-    index_path = cached_index_path(
-        trust_domain='gecko',
-        level=3,
-        cache_type='docker-images.v1',
-        cache_name=image_name,
-        digest=context_hash,
-    )
-    task_id = find_task_id(index_path)
+    if task_id in (True, False):
+        print(
+            'Could not find artifacts for a docker image '
+            'named `{image_name}`. Local commits and other changes '
+            'in your checkout may cause this error. Try '
+            'updating to a fresh checkout of mozilla-central '
+            'to download image.'.format(image_name=image_name))
+        return False
 
     return load_image_by_task_id(task_id, tag)
 
 
 def load_image_by_task_id(task_id, tag=None):
     artifact_url = get_artifact_url(task_id, 'public/image.tar.zst')
     result = load_image(artifact_url, tag)
     print("Found docker image: {}:{}".format(result['image'], result['tag']))
--- a/taskcluster/taskgraph/util/cached_tasks.py
+++ b/taskcluster/taskgraph/util/cached_tasks.py
@@ -65,36 +65,8 @@ def add_optimization(config, taskdesc, c
                                             time.gmtime(config.params['build_date']))
     taskdesc['routes'].extend([
         'index.{}'.format(route.format(**subs))
         for route in EXTRA_CACHE_INDEXES
     ])
 
     taskdesc['attributes']['cache_digest'] = digest
     taskdesc['attributes']['cache_type'] = cache_type
-
-
-def cached_index_path(level, trust_domain, cache_type, cache_name, digest=None, digest_data=None):
-    """
-    Get the index path needed to locate the task that would be created by
-    :func:`add_optimization`.
-
-    :param int level: The SCM level of the task to look for.
-    :param str trust_domain: The trust domain to look for the task in.
-    :param str cache_type: The type of task result being cached.
-    :param str cache_name: The name of the object being cached.
-    :param digest: A unique string indentifying this version of the artifacts
-        being generated. Typically this will be the hash of inputs to the task.
-    :type digest: bytes or None
-    :param digest_data: A list of bytes representing the inputs of this task.
-        They will be concatenated and hashed to create the digest for this
-        task.
-    :type digest_data: list of bytes or None
-
-    :return str: The index path.
-    """
-    if (digest is None) == (digest_data is None):
-        raise Exception("Must pass exactly one of `digest` and `digest_data`.")
-    if digest is None:
-        digest = hashlib.sha256('\n'.join(digest_data)).hexdigest()
-
-    return TARGET_CACHE_INDEX.format(
-        trust_domain=trust_domain, level=level, type=cache_type, name=cache_name, digest=digest)