Bug 1288567 - Use context hash of tar file; r=dustin
authorGregory Szorc <gps@mozilla.com>
Fri, 22 Jul 2016 10:24:08 -0700
changeset 348595 df2587962e55286f9f46070df84620fc92d3cf50
parent 348594 ce408d28753e15e523a9a3b3e2b4404dd1a88836
child 348596 f13b4d398a1b6df8c19216994194c073dd6e188b
push id1230
push userjlund@mozilla.com
push dateMon, 31 Oct 2016 18:13:35 +0000
treeherdermozilla-release@5e06e3766db2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1288567
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1288567 - Use context hash of tar file; r=dustin Now that tar file generation is deterministic, we can use the hash of the created archive rather than the hash of the files that are (presumably) in the archive. This temporarily breaks consistent hashing by using independent hashing mechanisms. This will be cleaned up in a subsequent commit. MozReview-Commit-ID: CWooVGfDKZO
taskcluster/taskgraph/task/docker_image.py
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -59,38 +59,40 @@ class DockerImageTask(base.Task):
             'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
                       .format(repo=params['head_repository'], rev=params['head_rev']),
         }
 
         tasks = []
         templates = Templates(path)
         for image_name in config['images']:
             context_path = os.path.join('testing', 'docker', image_name)
-            context_hash = generate_context_hash(context_path)
 
             image_parameters = dict(parameters)
-            image_parameters['context_hash'] = context_hash
             image_parameters['context_path'] = context_path
             image_parameters['artifact_path'] = 'public/image.tar'
             image_parameters['image_name'] = image_name
 
             image_artifact_path = \
                 "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name)
             if os.environ.get('TASK_ID'):
                 destination = os.path.join(
                     os.environ['HOME'],
                     "artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name))
                 image_parameters['context_url'] = ARTIFACT_URL.format(
                     os.environ['TASK_ID'], image_artifact_path)
-                cls.create_context_tar(context_path, destination, image_name)
+                context_hash = cls.create_context_tar(context_path, destination,
+                                                      image_name)
             else:
                 # skip context generation since this isn't a decision task
                 # TODO: generate context tarballs using subdirectory clones in
                 # the image-building task so we don't have to worry about this.
                 image_parameters['context_url'] = 'file:///tmp/' + image_artifact_path
+                context_hash = generate_context_hash(context_path)
+
+            image_parameters['context_hash'] = context_hash
 
             image_task = templates.load('image.yml', image_parameters)
 
             attributes = {'image_name': image_name}
 
             # As an optimization, if the context hash exists for mozilla-central, that image
             # task ID will be used.  The reasoning behind this is that eventually everything ends
             # up on mozilla-central at some point if most tasks use this as a common image
@@ -128,22 +130,25 @@ class DockerImageTask(base.Task):
                 return True, existing_task['taskId']
             except urllib2.HTTPError:
                 pass
 
         return False, None
 
     @classmethod
     def create_context_tar(cls, context_dir, destination, image_name):
-        'Creates a tar file of a particular context directory.'
+        """Creates a tar file of a particular context directory.
+
+        Returns the SHA-256 hex digest of the created file.
+        """
         destination = os.path.abspath(destination)
         if not os.path.exists(os.path.dirname(destination)):
             os.makedirs(os.path.dirname(destination))
 
-        create_context_tar(context_dir, destination, image_name)
+        return create_context_tar(context_dir, destination, image_name)
 
     @classmethod
     def from_json(cls, task_dict):
         # Generating index_paths for optimization
         routes = task_dict['task']['routes']
         index_paths = []
         for route in routes:
             index_path_regex = re.compile(INDEX_REGEX)