Bug 1333604 - Allow a treeherder instance to be passed to actions
authorBrian Stack <bstack@mozilla.com>
Wed, 25 Jan 2017 15:48:52 -0800
changeset 944289 986242e3af58d056b52c3ffa5505adfbbdb65953
parent 944018 24d9eb148461bb4789848b9880867c63c783a2ca
child 944290 d2a87d8c65ec142b6125ea0012472f21cb3dfdc1
push id166213
push userbstack@mozilla.com
push dateWed, 25 Jan 2017 23:55:13 +0000
treeherdertry@d2a87d8c65ec [default view] [failures only]
bugs1333604
milestone54.0a1
Bug 1333604 - Allow a treeherder instance to be passed to actions MozReview-Commit-ID: 6HUACMZfu7q
taskcluster/mach_commands.py
taskcluster/taskgraph/action.py
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -220,29 +220,34 @@ class MachCommands(MachCommandBase):
     @SubCommand('taskgraph', 'backfill',
                 description="Run the backfill task")
     @CommandArgument('--root', '-r',
                      default='taskcluster/ci',
                      help="root of the taskgraph definition relative to topsrcdir")
     @CommandArgument('--project',
                      required=True,
                      help="Project of the jobs that need to be backfilled.")
+    @CommandArgument('--treeherder-url',
+                     required=False,
+                     default='https://treeherder.mozilla.org/api',
+                     help="A treeherder instance to run this backfill against.")
     @CommandArgument('--job-id',
                      required=True,
                      help="Id of the job to be backfilled.")
     def taskgraph_backfill(self, **options):
         """Run the backfill task: Given a job in a project, it will
         add that job type to any previous revisions in treeherder
         until either a hard limit is met or a green version of that
         job is found."""
 
         import taskgraph.action
         try:
             self.setup_logging()
-            return taskgraph.action.backfill(options['project'], options['job_id'])
+            return taskgraph.action.backfill(options['project'],
+                                             options['job_id'], options['treeherder_url'])
         except Exception:
             traceback.print_exc()
             sys.exit(1)
 
     @SubCommand('taskgraph', 'cron',
                 description="Run the cron task")
     @CommandArgument('--base-repository',
                      required=True,
--- a/taskcluster/taskgraph/action.py
+++ b/taskcluster/taskgraph/action.py
@@ -13,17 +13,16 @@ import yaml
 
 from .create import create_tasks
 from .decision import write_artifact
 from .optimize import optimize_task_graph
 from .taskgraph import TaskGraph
 
 logger = logging.getLogger(__name__)
 TASKCLUSTER_QUEUE_URL = "https://queue.taskcluster.net/v1/task"
-TREEHERDER_URL = "https://treeherder.mozilla.org/api"
 
 # We set this to 5 for now because this is what SETA sets the
 # count to for every repository/job. If this is ever changed,
 # we'll need to have an API added to Treeherder to let us query
 # how far back we should look.
 MAX_BACKFILL_RESULTSETS = 5
 
 
@@ -67,64 +66,68 @@ def get_artifact(task_id, path):
     resp = requests.get(url="{}/{}/artifacts/{}".format(TASKCLUSTER_QUEUE_URL, task_id, path))
     if path.endswith('.json'):
         artifact = json.loads(resp.text)
     elif path.endswith('.yml'):
         artifact = yaml.load(resp.text)
     return artifact
 
 
-def backfill(project, job_id):
+def backfill(project, job_id, treeherder_url):
     """
     Run the backfill task.  This function implements `mach taskgraph backfill-task`,
     and is responsible for
 
      * Scheduling backfill jobs from a given treeherder resultset backwards until either
      a successful job is found or `N` jobs have been scheduled.
     """
     s = requests.Session()
     s.headers.update({"User-Agent": "gecko-intree-backfill-task"})
 
-    job = s.get(url="{}/project/{}/jobs/{}/".format(TREEHERDER_URL, project, job_id)).json()
+    job = s.get(url="{}/project/{}/jobs/{}/".format(treeherder_url, project, job_id))
+    if not job.ok:
+        logger.exception(job.text)
+        return
+    job = job.json()
 
     if job["build_system_type"] != "taskcluster":
         logger.warning("Invalid build system type! Must be a Taskcluster job. Aborting.")
         return
 
     filters = dict((k, job[k]) for k in ("build_platform_id", "platform_option", "job_type_id"))
 
-    resultset_url = "{}/project/{}/resultset/".format(TREEHERDER_URL, project)
+    resultset_url = "{}/project/{}/resultset/".format(treeherder_url, project)
     params = {"id__lt": job["result_set_id"], "count": MAX_BACKFILL_RESULTSETS}
     results = s.get(url=resultset_url, params=params).json()["results"]
     resultsets = [resultset["id"] for resultset in results]
 
-    for decision in load_decisions(s, project, resultsets, filters):
+    for decision in load_decisions(s, project, resultsets, filters, treeherder_url):
         add_tasks(decision, [job["job_type_name"]], '{}-'.format(decision))
 
 
 def add_talos(decision_task_id, times=1):
     """
     Run the add-talos task.  This function implements `mach taskgraph add-talos`,
     and is responsible for
 
      * Adding all talos jobs to a push.
     """
     full_task_json = get_artifact(decision_task_id, "public/full-task-graph.json")
     task_labels = [label for label in full_task_json if "talos" in label]
     for time in xrange(times):
         add_tasks(decision_task_id, task_labels, '{}-'.format(time))
 
 
-def load_decisions(s, project, resultsets, filters):
+def load_decisions(s, project, resultsets, filters, treeherder_url):
     """
     Given a project, a list of revisions, and a dict of filters, return
     a list of taskIds from decision tasks.
     """
-    project_url = "{}/project/{}/jobs/".format(TREEHERDER_URL, project)
-    decision_url = "{}/jobdetail/".format(TREEHERDER_URL)
+    project_url = "{}/project/{}/jobs/".format(treeherder_url, project)
+    decision_url = "{}/jobdetail/".format(treeherder_url)
     decisions = []
     decision_ids = []
 
     for resultset in resultsets:
         unfiltered = []
         offset = 0
         jobs_per_call = 250
         while True: