Bug 1648723 - Make it possible to run the bugbug optimization strategy on a set of pushes. r=ahal
authorMarco Castelluccio <mcastelluccio@mozilla.com>
Thu, 30 Jul 2020 17:13:56 +0000
changeset 542643 b99b7e7967bb375d4f2ee1c42a9085c76d8a42e8
parent 542642 8aec19091f93f05f9bb97b9386ed98f379474acc
child 542644 55bcd608e8340f26d69309ae72d52df4f9f3225b
push id37653
push userbtara@mozilla.com
push dateThu, 30 Jul 2020 21:54:52 +0000
treeherdermozilla-central@c34351a5fd6c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1648723
milestone81.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1648723 - Make it possible to run the bugbug optimization strategy on a set of pushes. r=ahal Differential Revision: https://phabricator.services.mozilla.com/D85275
taskcluster/taskgraph/optimize/bugbug.py
taskcluster/taskgraph/test/test_optimize_strategies.py
taskcluster/taskgraph/util/hg.py
--- a/taskcluster/taskgraph/optimize/bugbug.py
+++ b/taskcluster/taskgraph/optimize/bugbug.py
@@ -11,73 +11,110 @@ from six.moves.urllib.parse import urlsp
 from taskgraph.optimize import register_strategy, OptimizationStrategy, seta
 from taskgraph.util.bugbug import (
     BugbugTimeoutException,
     push_schedules,
     CT_HIGH,
     CT_MEDIUM,
     CT_LOW,
 )
+from taskgraph.util.hg import get_push_data
 
 
 @register_strategy("bugbug-low", args=(CT_LOW,))
 @register_strategy("bugbug-medium", args=(CT_MEDIUM,))
 @register_strategy("bugbug-high", args=(CT_HIGH,))
 @register_strategy("bugbug-tasks-medium", args=(CT_MEDIUM, True))
 @register_strategy("bugbug-tasks-high", args=(CT_HIGH, True))
 @register_strategy("bugbug-reduced", args=(CT_MEDIUM, True, True))
 @register_strategy("bugbug-reduced-fallback", args=(CT_MEDIUM, True, True, True))
 @register_strategy("bugbug-reduced-high", args=(CT_HIGH, True, True))
 @register_strategy("bugbug-reduced-manifests", args=(CT_MEDIUM, False, True))
 @register_strategy("bugbug-reduced-manifests-fallback", args=(CT_MEDIUM, False, True, True))
+@register_strategy("bugbug-reduced-fallback-last-10-pushes", args=(0.3, False, True, True, 10))
 class BugBugPushSchedules(OptimizationStrategy):
     """Query the 'bugbug' service to retrieve relevant tasks and manifests.
 
     Args:
         confidence_threshold (float): The minimum confidence threshold (in
             range [0, 1]) needed for a task to be scheduled.
         tasks_only (bool): Whether or not to only use tasks and no groups
             (default: False)
         use_reduced_tasks (bool): Whether or not to use the reduced set of tasks
             provided by the bugbug service (default: False).
         fallback (bool): Whether or not to fallback to SETA if there was a failure
             in bugbug (default: False)
+        num_pushes (int): The number of pushes to consider for the selection
+            (default: 1).
     """
 
     def __init__(
         self,
         confidence_threshold,
         tasks_only=False,
         use_reduced_tasks=False,
         fallback=False,
+        num_pushes=1,
     ):
         self.confidence_threshold = confidence_threshold
         self.use_reduced_tasks = use_reduced_tasks
         self.fallback = fallback
         self.tasks_only = tasks_only
+        self.num_pushes = num_pushes
         self.timedout = False
 
     def should_remove_task(self, task, params, importance):
-        if params['project'] not in ("autoland", "try"):
+        project = params['project']
+
+        if project not in ("autoland", "try"):
             return False
 
+        current_push_id = int(params['pushlog_id'])
+
         branch = urlsplit(params['head_repository']).path.strip('/')
         rev = params['head_rev']
 
         if self.timedout:
-            return seta.is_low_value_task(task.label, params['project'])
+            return seta.is_low_value_task(task.label, project)
+
+        data = {}
+
+        start_push_id = current_push_id - self.num_pushes + 1
+        if self.num_pushes != 1:
+            push_data = get_push_data(
+                params["head_repository"], project, start_push_id, current_push_id - 1
+            )
+
+        for push_id in range(start_push_id, current_push_id + 1):
+            if push_id == current_push_id:
+                rev = params["head_rev"]
+            else:
+                rev = push_data[push_id]["changesets"][-1]
 
-        try:
-            data = push_schedules(branch, rev)
-        except BugbugTimeoutException:
-            if not self.fallback:
-                raise
+            try:
+                new_data = push_schedules(branch, rev)
+                for key, value in new_data.items():
+                    if isinstance(value, dict):
+                        if key not in data:
+                            data[key] = {}
 
-            self.timedout = True
-            return self.should_remove_task(task, params, importance)
+                        for name, confidence in value.items():
+                            if name not in data[key] or data[key][name] < confidence:
+                                data[key][name] = confidence
+                    elif isinstance(value, list):
+                        if key not in data:
+                            data[key] = set()
+
+                        data[key].update(value)
+            except BugbugTimeoutException:
+                if not self.fallback:
+                    raise
+
+                self.timedout = True
+                return self.should_remove_task(task, params, importance)
 
         key = "reduced_tasks" if self.use_reduced_tasks else "tasks"
         tasks = set(
             task
             for task, confidence in data.get(key, {}).items()
             if confidence >= self.confidence_threshold
         )
 
--- a/taskcluster/taskgraph/test/test_optimize_strategies.py
+++ b/taskcluster/taskgraph/test/test_optimize_strategies.py
@@ -215,16 +215,58 @@ def test_bugbug_push_schedules(responses
         status=200,
     )
 
     opt = BugBugPushSchedules(*args)
     labels = [t.label for t in default_tasks if not opt.should_remove_task(t, params, {})]
     assert sorted(labels) == sorted(expected)
 
 
+def test_bugbug_multiple_pushes(responses, params):
+    pushes = {str(pid): {"changesets": ["c{}".format(pid)]} for pid in range(8, 10)}
+
+    responses.add(
+        responses.GET,
+        "https://hg.mozilla.org/integration/autoland/json-pushes/?version=2&startID=8&endID=9",
+        json={"pushes": pushes},
+        status=200,
+    )
+
+    responses.add(
+        responses.GET,
+        BUGBUG_BASE_URL + "/push/{}/c9/schedules".format(params["branch"]),
+        json={
+            'tasks': {'task-2': 0.2, 'task-4': 0.5},
+            'groups': {'foo/test.ini': 0.2, 'bar/test.ini': 0.25},
+            'known_tasks': ['task-4'],
+        },
+        status=200,
+    )
+
+    # Tasks with a lower confidence don't override task with a higher one.
+    # Tasks with a higher confidence override tasks with a lower one.
+    # Known tasks are merged.
+    responses.add(
+        responses.GET,
+        BUGBUG_BASE_URL + "/push/{branch}/{head_rev}/schedules".format(**params),
+        json={
+            'tasks': {'task-2': 0.2, 'task-4': 0.2},
+            'groups': {'foo/test.ini': 0.65, 'bar/test.ini': 0.25},
+            'known_tasks': ['task-1', 'task-3'],
+        },
+        status=200,
+    )
+
+    params["pushlog_id"] = 10
+
+    opt = BugBugPushSchedules(0.3, False, False, False, 2)
+    labels = [t.label for t in default_tasks if not opt.should_remove_task(t, params, {})]
+    assert sorted(labels) == sorted(['task-0', 'task-2', 'task-4'])
+
+
 def test_bugbug_timeout(monkeypatch, responses, params):
     query = "/push/{branch}/{head_rev}/schedules".format(**params)
     url = BUGBUG_BASE_URL + query
     responses.add(
         responses.GET,
         url,
         json={"ready": False},
         status=202,
--- a/taskcluster/taskgraph/util/hg.py
+++ b/taskcluster/taskgraph/util/hg.py
@@ -1,56 +1,105 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import logging
+
 import requests
 import six
 import subprocess
 from redo import retry
 
-PUSHLOG_TMPL = '{}/json-pushes?version=2&changeset={}&tipsonly=1'
+from mozbuild.util import memoize
+
+logger = logging.getLogger(__name__)
+
+PUSHLOG_CHANGESET_TMPL = (
+    "{repository}/json-pushes?version=2&changeset={revision}&tipsonly=1"
+)
+PUSHLOG_PUSHES_TMPL = (
+    "{repository}/json-pushes/?version=2&startID={push_id_start}&endID={push_id_end}"
+)
+
+
+def _query_pushlog(url):
+    response = retry(requests.get, attempts=5, sleeptime=10,
+                     args=(url, ),
+                     kwargs={'timeout': 60, 'headers': {'User-Agent': 'TaskCluster'}})
+
+    return response.json()["pushes"]
 
 
 def find_hg_revision_push_info(repository, revision):
     """Given the parameters for this action and a revision, find the
     pushlog_id of the revision."""
-    pushlog_url = PUSHLOG_TMPL.format(repository, revision)
+    url = PUSHLOG_CHANGESET_TMPL.format(
+        repository=repository, revision=revision
+    )
 
-    def extract_pushes(response_json):
-        pushes = response_json['pushes']
-        if len(pushes) != 1:
-            raise RuntimeError(
-                "Found {} pushlog_ids, expected 1, for {} revision {}: {}".format(
-                    len(pushes), repository, revision, pushes
-                )
+    pushes = _query_pushlog(url)
+
+    if len(pushes) != 1:
+        raise RuntimeError(
+            "Found {} pushlog_ids, expected 1, for {} revision {}: {}".format(
+                len(pushes), repository, revision, pushes
             )
-        return pushes
+        )
 
-    def query_pushlog(url):
-        r = requests.get(pushlog_url, timeout=60)
-        r.raise_for_status()
-        return extract_pushes(r.json())
-
-    pushes = retry(
-        query_pushlog, args=(pushlog_url,),
-        attempts=5, sleeptime=10,
-    )
     pushid = list(pushes.keys())[0]
     return {
         'pushdate': pushes[pushid]['date'],
         'pushid': pushid,
         'user': pushes[pushid]['user'],
     }
 
 
+@memoize
+def get_push_data(repository, project, push_id_start, push_id_end):
+    url = PUSHLOG_PUSHES_TMPL.format(
+        repository=repository,
+        push_id_start=push_id_start - 1,
+        push_id_end=push_id_end,
+    )
+
+    try:
+        pushes = _query_pushlog(url)
+
+        return {push_id: pushes[str(push_id)] for push_id in range(push_id_start, push_id_end + 1)}
+
+    # In the event of request times out, requests will raise a TimeoutError.
+    except requests.exceptions.Timeout:
+        logger.warning("json-pushes timeout")
+
+    # In the event of a network problem (e.g. DNS failure, refused connection, etc),
+    # requests will raise a ConnectionError.
+    except requests.exceptions.ConnectionError:
+        logger.warning("json-pushes connection error")
+
+    # In the event of the rare invalid HTTP response(e.g 404, 401),
+    # requests will raise an HTTPError exception
+    except requests.exceptions.HTTPError:
+        logger.warning("Bad Http response")
+
+    # When we get invalid JSON (i.e. 500 error), it results in a ValueError (bug 1313426)
+    except ValueError as error:
+        logger.warning("Invalid JSON, possible server error: {}".format(error))
+
+    # We just print the error out as a debug message if we failed to catch the exception above
+    except requests.exceptions.RequestException as error:
+        logger.warning(error)
+
+    return None
+
+
 def get_hg_revision_branch(root, revision):
     """Given the parameters for a revision, find the hg_branch (aka
     relbranch) of the revision."""
     return six.ensure_text(subprocess.check_output([
         'hg', 'identify',
         '-T', '{branch}',
         '--rev', revision,
     ], cwd=root, universal_newlines=True))