bug 1442793 - generate_taskcluster_artifact_prefix. r=bhearsum
authorAki Sasaki <asasaki@mozilla.com>
Wed, 21 Mar 2018 12:07:01 -0700
changeset 467496 58b09a0406d5581ee0e179cdf5f0164ac47b5469
parent 467495 d1c361a5ddc715b52adb3e1d8ad338ecdc59055d
child 467497 d552d6d9ac7f4b989541f723c5211336a0a90ce4
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbhearsum
bugs1442793
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1442793 - generate_taskcluster_artifact_prefix. r=bhearsum MozReview-Commit-ID: 2zRqm5C9ZZQ
taskcluster/taskgraph/transforms/final_verify.py
taskcluster/taskgraph/transforms/partials.py
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/update_verify.py
taskcluster/taskgraph/util/taskcluster.py
--- a/taskcluster/taskgraph/transforms/final_verify.py
+++ b/taskcluster/taskgraph/transforms/final_verify.py
@@ -20,17 +20,17 @@ def add_command(config, tasks):
         if not task["worker"].get("env"):
             task["worker"]["env"] = {}
 
         final_verify_configs = []
         for upstream in task.get("dependencies", {}).keys():
             if 'update-verify-config' in upstream:
                 final_verify_configs.append(
                     "{}update-verify.cfg".format(
-                        get_taskcluster_artifact_prefix("<{}>".format(upstream))
+                        get_taskcluster_artifact_prefix(task, "<{}>".format(upstream))
                     )
                 )
         task["worker"]["command"] = [
             "/bin/bash",
             "-c",
             {
                 "task-reference": "hg clone $BUILD_TOOLS_REPO tools && cd tools/release && " +
                                   "./final-verification.sh " +
--- a/taskcluster/taskgraph/transforms/partials.py
+++ b/taskcluster/taskgraph/transforms/partials.py
@@ -83,17 +83,17 @@ def make_task_description(config, jobs):
             if 'repackage-signing' in dependency:
                 signing_task = dependency
                 break
         signing_task_ref = '<{}>'.format(signing_task)
 
         extra = {'funsize': {'partials': list()}}
         update_number = 1
         artifact_path = "{}{}".format(
-            get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
+            get_taskcluster_artifact_prefix(dep_job, signing_task_ref, locale=locale),
             'target.complete.mar'
         )
         for build in sorted(builds):
             partial_info = {
                 'locale': build_locale,
                 'from_mar': builds[build]['mar_url'],
                 'to_mar': {'task-reference': artifact_path},
                 'platform': get_balrog_platform_name(dep_th_platform),
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -179,17 +179,17 @@ def make_job_description(config, jobs):
             'using': 'mozharness',
             'script': 'mozharness/scripts/repackage.py',
             'job-script': 'taskcluster/scripts/builder/repackage.sh',
             'actions': ['download_input', 'setup', 'repackage'],
             'extra-workspace-cache-key': 'repackage',
         })
 
         worker = {
-            'env': _generate_task_env(build_platform, build_task_ref,
+            'env': _generate_task_env(dep_job, build_platform, build_task_ref,
                                       signing_task_ref, locale=locale),
             'artifacts': _generate_task_output_files(build_platform, locale=locale),
             'chain-of-trust': True,
             'max-run-time': 7200 if build_platform.startswith('win') else 3600,
         }
 
         if locale:
             # Make sure we specify the locale-specific upload dir
@@ -237,19 +237,21 @@ def make_job_description(config, jobs):
         if build_platform.startswith('macosx'):
             task['toolchains'] = [
                 'linux64-libdmg',
                 'linux64-hfsplus',
             ]
         yield task
 
 
-def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
-    mar_prefix = get_taskcluster_artifact_prefix(build_task_ref, postfix='host/bin/', locale=None)
-    signed_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale)
+def _generate_task_env(task, build_platform, build_task_ref, signing_task_ref, locale=None):
+    mar_prefix = get_taskcluster_artifact_prefix(
+        task, build_task_ref, postfix='host/bin/', locale=None
+    )
+    signed_prefix = get_taskcluster_artifact_prefix(task, signing_task_ref, locale=locale)
 
     if build_platform.startswith('linux') or build_platform.startswith('macosx'):
         tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
         return {
             'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format(
                 signed_prefix, tarball_extension
             )},
             'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
--- a/taskcluster/taskgraph/transforms/update_verify.py
+++ b/taskcluster/taskgraph/transforms/update_verify.py
@@ -42,17 +42,17 @@ def add_command(config, tasks):
             for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
                 thing = "worker.env.{}".format(thing)
                 resolve_keyed_by(chunked, thing, thing, **config.params)
 
             update_verify_config = None
             for upstream in chunked.get("dependencies", {}).keys():
                 if 'update-verify-config' in upstream:
                     update_verify_config = "{}update-verify.cfg".format(
-                        get_taskcluster_artifact_prefix("<{}>".format(upstream))
+                        get_taskcluster_artifact_prefix(task, "<{}>".format(upstream))
                     )
             if not update_verify_config:
                 raise Exception("Couldn't find upate verify config")
 
             chunked["worker"]["env"]["TASKCLUSTER_VERIFY_CONFIG"] = {
                 "task-reference": update_verify_config
             }
 
--- a/taskcluster/taskgraph/util/taskcluster.py
+++ b/taskcluster/taskgraph/util/taskcluster.py
@@ -12,17 +12,17 @@ import yaml
 import requests
 import logging
 from mozbuild.util import memoize
 from requests.packages.urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter
 from taskgraph.task import Task
 
 _TC_ARTIFACT_LOCATION = \
-        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
+        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}'
 
 logger = logging.getLogger(__name__)
 
 # this is set to true for `mach taskgraph action-callback --test`
 testing = False
 
 
 @memoize
@@ -204,13 +204,17 @@ def purge_cache(provisioner_id, worker_t
     if testing:
         logger.info('Would have purged {}/{}/{}.'.format(provisioner_id, worker_type, cache_name))
     else:
         logger.info('Purging {}/{}/{}.'.format(provisioner_id, worker_type, cache_name))
         purge_cache_url = get_purge_cache_url(provisioner_id, worker_type, use_proxy)
         _do_request(purge_cache_url, json={'cacheName': cache_name})
 
 
-def get_taskcluster_artifact_prefix(task_id, postfix='', locale=None):
+def get_taskcluster_artifact_prefix(task, task_id, postfix='', locale=None):
     if locale:
         postfix = '{}/{}'.format(locale, postfix)
 
-    return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)
+    artifact_prefix = get_artifact_prefix(task)
+
+    return _TC_ARTIFACT_LOCATION.format(
+        task_id=task_id, postfix=postfix, artifact_prefix=artifact_prefix
+    )