Bug 1288567 - Use context hash of tar file; r=dustin
authorGregory Szorc <gps@mozilla.com>
Fri, 22 Jul 2016 10:24:08 -0700
changeset 331559 df2587962e55286f9f46070df84620fc92d3cf50
parent 331558 ce408d28753e15e523a9a3b3e2b4404dd1a88836
child 331560 f13b4d398a1b6df8c19216994194c073dd6e188b
push id9858
push userjlund@mozilla.com
push dateMon, 01 Aug 2016 14:37:10 +0000
treeherdermozilla-aurora@203106ef6cb6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
Bug 1288567 - Use context hash of tar file; r=dustin Now that tar file generation is deterministic, we can use the hash of the created archive rather than the hash of the files that are (presumably) in the archive. This temporarily breaks consistent hashing by using independent hashing mechanisms. This will be cleaned up in a subsequent commit. MozReview-Commit-ID: CWooVGfDKZO
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -59,38 +59,40 @@ class DockerImageTask(base.Task):
             'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
                       .format(repo=params['head_repository'], rev=params['head_rev']),
         tasks = []
         templates = Templates(path)
         for image_name in config['images']:
             context_path = os.path.join('testing', 'docker', image_name)
-            context_hash = generate_context_hash(context_path)
             image_parameters = dict(parameters)
-            image_parameters['context_hash'] = context_hash
             image_parameters['context_path'] = context_path
             image_parameters['artifact_path'] = 'public/image.tar'
             image_parameters['image_name'] = image_name
             image_artifact_path = \
             if os.environ.get('TASK_ID'):
                 destination = os.path.join(
                 image_parameters['context_url'] = ARTIFACT_URL.format(
                     os.environ['TASK_ID'], image_artifact_path)
-                cls.create_context_tar(context_path, destination, image_name)
+                context_hash = cls.create_context_tar(context_path, destination,
+                                                      image_name)
                 # skip context generation since this isn't a decision task
                 # TODO: generate context tarballs using subdirectory clones in
                 # the image-building task so we don't have to worry about this.
                 image_parameters['context_url'] = 'file:///tmp/' + image_artifact_path
+                context_hash = generate_context_hash(context_path)
+            image_parameters['context_hash'] = context_hash
             image_task = templates.load('image.yml', image_parameters)
             attributes = {'image_name': image_name}
             # As an optimization, if the context hash exists for mozilla-central, that image
             # task ID will be used.  The reasoning behind this is that eventually everything ends
             # up on mozilla-central at some point if most tasks use this as a common image
@@ -128,22 +130,25 @@ class DockerImageTask(base.Task):
                 return True, existing_task['taskId']
             except urllib2.HTTPError:
         return False, None
     def create_context_tar(cls, context_dir, destination, image_name):
-        'Creates a tar file of a particular context directory.'
+        """Creates a tar file of a particular context directory.
+        Returns the SHA-256 hex digest of the created file.
+        """
         destination = os.path.abspath(destination)
         if not os.path.exists(os.path.dirname(destination)):
-        create_context_tar(context_dir, destination, image_name)
+        return create_context_tar(context_dir, destination, image_name)
     def from_json(cls, task_dict):
         # Generating index_paths for optimization
         routes = task_dict['task']['routes']
         index_paths = []
         for route in routes:
             index_path_regex = re.compile(INDEX_REGEX)