Bug 1352355 - Enable storing and updating mozleak metadata in wpt ini, r=ato
authorJames Graham <james@hoppipolla.co.uk>
Fri, 30 Nov 2018 23:17:39 +0000
changeset 508569 02e4dbfecbc426e118199e4ce140db13814aaf7a
parent 508568 80a92104544fbab8e6506c86e6b1230a1da01dd4
child 508570 d40ebdfc91a0f84e73af05cbf4335b9d3adcb33d
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato
bugs1352355
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1352355 - Enable storing and updating mozleak metadata in wpt ini, r=ato This adds two new properties to wpt metadata files: mozleak-allowed - This is a list of the form [process-name:object name], which indicates objects that may be leaked in that specific process. Automatic updates that find a leak may add that object to this list. mozleak-threshold - This is a list (but conceptually a map) of [process-name: threshold bytes], indicating a threshold below which leaks will not cause a test failure. This number is updated by setting it to the observed value for a process. MozReview-Commit-ID: KA1oPl837a8 Depends on D12410 Differential Revision: https://phabricator.services.mozilla.com/D12411
testing/web-platform/meta/__dir__.ini
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
testing/web-platform/tests/tools/wptrunner/wptrunner/formatters.py
testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
testing/web-platform/tests/tools/wptrunner/wptrunner/manifestupdate.py
testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_update.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py
--- a/testing/web-platform/meta/__dir__.ini
+++ b/testing/web-platform/meta/__dir__.ini
@@ -1,1 +1,3 @@
 lsan-allowed: [js_pod_malloc, js_pod_calloc, js_pod_realloc, js_arena_calloc,js_pod_arena_calloc, maybe_pod_calloc, pod_calloc, make_zeroed_pod_array, js_arena_malloc]
+leak-threshold: [tab:10000, geckomediaplugin:20000]
+
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -193,39 +193,47 @@ class FirefoxBrowser(Browser):
             self.stack_fixer = None
 
         if timeout_multiplier:
             self.init_timeout = self.init_timeout * timeout_multiplier
 
         self.asan = asan
         self.lsan_allowed = None
         self.lsan_max_stack_depth = None
+        self.mozleak_allowed = None
+        self.mozleak_thresholds = None
         self.leak_check = leak_check
         self.leak_report_file = None
         self.lsan_handler = None
         self.stylo_threads = stylo_threads
         self.chaos_mode_flags = chaos_mode_flags
         self.headless = headless
 
     def settings(self, test):
         self.lsan_allowed = test.lsan_allowed
+
+    def settings(self, test):
+        self.lsan_allowed = test.lsan_allowed
         self.lsan_max_stack_depth = test.lsan_max_stack_depth
+        self.mozleak_allowed = test.mozleak_allowed
+        self.mozleak_thresholds = test.mozleak_threshold
         return {"check_leaks": self.leak_check and not test.leaks,
                 "lsan_allowed": test.lsan_allowed}
 
     def start(self, group_metadata=None, **kwargs):
         if group_metadata is None:
             group_metadata = {}
 
+        self.group_metadata = group_metadata
+
         if self.marionette_port is None:
             self.marionette_port = get_free_port(2828, exclude=self.used_ports)
             self.used_ports.add(self.marionette_port)
 
         if self.asan:
-            print "Setting up LSAN"
             self.lsan_handler = mozleak.LSANLeaks(self.logger,
                                                   scope=group_metadata.get("scope", "/"),
                                                   allowed=self.lsan_allowed,
                                                   maxNumRecordedFrames=self.lsan_max_stack_depth)
 
         env = test_environment(xrePath=os.path.dirname(self.binary),
                                debugger=self.debug_info is not None,
                                log=self.logger,
@@ -336,31 +344,28 @@ class FirefoxBrowser(Browser):
                             break
             except OSError:
                 # This can happen on Windows if the process is already dead
                 pass
         self.process_leaks()
         self.logger.debug("stopped")
 
     def process_leaks(self):
-        self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file)
+        self.logger.info("PROCESS LEAKS %s" % self.leak_report_file)
         if self.lsan_handler:
             self.lsan_handler.process()
         if self.leak_report_file is not None:
             mozleak.process_leak_log(
                 self.leak_report_file,
-                leak_thresholds={
-                    "default": 0,
-                    "tab": 10000,  # See dependencies of bug 1051230.
-                    # GMP rarely gets a log, but when it does, it leaks a little.
-                    "geckomediaplugin": 20000,
-                },
+                leak_thresholds=self.mozleak_thresholds,
                 ignore_missing_leaks=["geckomediaplugin"],
                 log=self.logger,
-                stack_fixer=self.stack_fixer
+                stack_fixer=self.stack_fixer,
+                scope=self.group_metadata.get("scope"),
+                allowed=self.mozleak_allowed
             )
 
     def pid(self):
         if self.runner.process_handler is None:
             return None
 
         try:
             return self.runner.process_handler.pid
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/formatters.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/formatters.py
@@ -126,8 +126,29 @@ class WptreportFormatter(BaseFormatter):
 
     def lsan_leak(self, data):
         if "lsan_leaks" not in self.results:
             self.results["lsan_leaks"] = []
         lsan_leaks = self.results["lsan_leaks"]
         lsan_leaks.append({"frames": data["frames"],
                            "scope": data["scope"],
                            "allowed_match": data.get("allowed_match")})
+
+    def find_or_create_mozleak(self, data):
+        if "mozleak" not in self.results:
+            self.results["mozleak"] = {}
+        scope = data["scope"]
+        if scope not in self.results["mozleak"]:
+            self.results["mozleak"][scope] = {"objects": [], "total": []}
+        return self.results["mozleak"][scope]
+
+    def mozleak_object(self, data):
+        scope_data = self.find_or_create_mozleak(data)
+        scope_data["objects"].append({"process": data["process"],
+                                      "name": data["name"],
+                                      "allowed": data.get("allowed", False),
+                                      "bytes": data["bytes"]})
+
+    def mozleak_total(self, data):
+        scope_data = self.find_or_create_mozleak(data)
+        scope_data["total"].append({"bytes": data["bytes"],
+                                    "threshold": data.get("threshold", 0),
+                                    "process": data["process"]})
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
@@ -65,28 +65,42 @@ def prefs(node):
             rv = dict(value(node_prefs))
         else:
             rv = dict(value(item) for item in node_prefs)
     except KeyError:
         rv = {}
     return rv
 
 
-def lsan_allowed(node):
+def set_prop(name, node):
     try:
-        node_items = node.get("lsan-allowed")
+        node_items = node.get(name)
         if isinstance(node_items, (str, unicode)):
             rv = {node_items}
         else:
             rv = set(node_items)
     except KeyError:
         rv = set()
     return rv
 
 
+def leak_threshold(node):
+    rv = {}
+    try:
+        node_items = node.get("leak-threshold")
+        if isinstance(node_items, (str, unicode)):
+            node_items = [node_items]
+        for item in node_items:
+            process, value = item.rsplit(":", 1)
+            rv[process.strip()] = int(value.strip())
+    except KeyError:
+        pass
+    return rv
+
+
 class ExpectedManifest(ManifestItem):
     def __init__(self, name, test_path, url_base):
         """Object representing all the tests in a particular manifest
 
         :param name: Name of the AST Node associated with this object.
                      Should always be None since this should always be associated with
                      the root node of the AST.
         :param test_path: Path of the test file associated with this manifest.
@@ -149,17 +163,25 @@ class ExpectedManifest(ManifestItem):
         return tags(self)
 
     @property
     def prefs(self):
         return prefs(self)
 
     @property
     def lsan_allowed(self):
-        return lsan_allowed(self)
+        return set_prop("lsan-allowed", self)
+
+    @property
+    def leak_allowed(self):
+        return set_prop("leak-allowed", self)
+
+    @property
+    def leak_threshold(self):
+        return leak_threshold(self)
 
     @property
     def lsan_max_stack_depth(self):
         return int_prop("lsan-max-stack-depth", self)
 
 
 class DirectoryManifest(ManifestItem):
     @property
@@ -187,17 +209,25 @@ class DirectoryManifest(ManifestItem):
         return tags(self)
 
     @property
     def prefs(self):
         return prefs(self)
 
     @property
     def lsan_allowed(self):
-        return lsan_allowed(self)
+        return set_prop("lsan-allowed", self)
+
+    @property
+    def leak_allowed(self):
+        return set_prop("leak-allowed", self)
+
+    @property
+    def leak_threshold(self):
+        return leak_threshold(self)
 
     @property
     def lsan_max_stack_depth(self):
         return int_prop("lsan-max-stack-depth", self)
 
 class TestNode(ManifestItem):
     def __init__(self, name):
         """Tree node associated with a particular test in a manifest
@@ -251,17 +281,25 @@ class TestNode(ManifestItem):
         return tags(self)
 
     @property
     def prefs(self):
         return prefs(self)
 
     @property
     def lsan_allowed(self):
-        return lsan_allowed(self)
+        return set_prop("lsan-allowed", self)
+
+    @property
+    def leak_allowed(self):
+        return set_prop("leak-allowed", self)
+
+    @property
+    def leak_threshold(self):
+        return leak_threshold(self)
 
     @property
     def lsan_max_stack_depth(self):
         return int_prop("lsan-max-stack-depth", self)
 
     def append(self, node):
         """Add a subtest to the current test
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestupdate.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestupdate.py
@@ -1,12 +1,13 @@
 import itertools
 import os
 import urlparse
 from collections import namedtuple, defaultdict
+from math import ceil
 
 from wptmanifest.node import (DataNode, ConditionalNode, BinaryExpressionNode,
                               BinaryOperatorNode, VariableNode, StringNode, NumberNode,
                               UnaryExpressionNode, UnaryOperatorNode, KeyValueNode)
 from wptmanifest.backends import conditional
 from wptmanifest.backends.conditional import ManifestItem
 
 import expected
@@ -76,16 +77,18 @@ class ExpectedManifest(ManifestItem):
         self.test_path = test_path
         self.url_base = url_base
         assert self.url_base is not None
         self.modified = False
         self.boolean_properties = boolean_properties
         self.property_order = property_order
         self.update_properties = {
             "lsan": LsanUpdate(self),
+            "leak-object": LeakObjectUpdate(self),
+            "leak-threshold": LeakThresholdUpdate(self),
         }
 
     def append(self, child):
         ManifestItem.append(self, child)
         if child.id in self.child_map:
             print "Warning: Duplicate heading %s" % child.id
         self.child_map[child.id] = child
 
@@ -117,16 +120,34 @@ class ExpectedManifest(ManifestItem):
         """Set the result of the test in a particular run
 
         :param run_info: Dictionary of run_info parameters corresponding
                          to this run
         :param result: Lsan violations detected"""
 
         self.update_properties["lsan"].set(run_info, result)
 
+    def set_leak_object(self, run_info, result):
+        """Set the result of the test in a particular run
+
+        :param run_info: Dictionary of run_info parameters corresponding
+                         to this run
+        :param result: Leaked objects deletec"""
+
+        self.update_properties["leak-object"].set(run_info, result)
+
+    def set_leak_threshold(self, run_info, result):
+        """Set the result of the test in a particular run
+
+        :param run_info: Dictionary of run_info parameters corresponding
+                         to this run
+        :param result: Total number of bytes leaked"""
+
+        self.update_properties["leak-threshold"].set(run_info, result)
+
     def coalesce_properties(self, stability):
         for prop_update in self.update_properties.itervalues():
             prop_update.coalesce(stability)
 
 
 class TestNode(ManifestItem):
     def __init__(self, node):
         """Tree node associated with a particular test in a manifest
@@ -427,34 +448,35 @@ class ExpectedUpdate(PropertyUpdate):
             else:
                 raise UpdateError("Conflicting metadata values")
         for conditional_node, value in conditionals:
             if value != unconditional_value:
                 self.node.set(self.property_name, value, condition=conditional_node.children[0])
 
 
 class MaxAssertsUpdate(PropertyUpdate):
+    """For asserts we always update the default value and never add new conditionals.
+    The value we set as the default is the maximum the current default or one more than the
+    number of asserts we saw in any configuration."""
+
     property_name = "max-asserts"
     cls_default_value = 0
     value_type = int
 
     def update_value(self, old_value, new_value):
         new_value = self.value_type(new_value)
         if old_value is not None:
             old_value = self.value_type(old_value)
         if old_value is not None and old_value < new_value:
             return new_value + 1
         if old_value is None:
             return new_value + 1
         return old_value
 
     def update_default(self):
-        """For asserts we always update the default value and never add new conditionals.
-        The value we set as the default is the maximum the current default or one more than the
-        number of asserts we saw in any configuration."""
         # Current values
         values = []
         current_default = None
         if self.property_name in self.node._data:
             current_default = [item for item in
                                self.node._data[self.property_name]
                                if item.condition_node is None]
             if current_default:
@@ -496,30 +518,21 @@ class MinAssertsUpdate(PropertyUpdate):
             values.append(current_default[0].value_as(self.value_type))
         values.extend(max(0, item.value) for item in self.new)
         values.extend(max(0, item.value) for item in
                       itertools.chain.from_iterable(results for _, results in self.updated))
         new_value = min(values)
         return True, new_value
 
 
-class LsanUpdate(PropertyUpdate):
-    property_name = "lsan-allowed"
+class AppendOnlyListUpdate(PropertyUpdate):
     cls_default_value = None
 
     def get_value(self, result):
-        # If we have an allowed_match that matched, return None
-        # This value is ignored later (because it matches the default)
-        # We do that because then if we allow a failure in foo/__dir__.ini
-        # we don't want to update foo/bar/__dir__.ini with the same rule
-        if result[1]:
-            return None
-        # Otherwise return the topmost stack frame
-        # TODO: there is probably some improvement to be made by looking for a "better" stack frame
-        return result[0][0]
+        raise NotImplementedError
 
     def update_value(self, old_value, new_value):
         if isinstance(new_value, (str, unicode)):
             new_value = {new_value}
         else:
             new_value = set(new_value)
         if old_value is None:
             old_value = set()
@@ -534,16 +547,106 @@ class LsanUpdate(PropertyUpdate):
                                if item.condition_node is None]
         if current_default:
             current_default = current_default[0].value
         new_values = [item.value for item in self.new]
         new_value = self.update_value(current_default, new_values)
         return True, new_value if new_value else None
 
 
+class LsanUpdate(AppendOnlyListUpdate):
+    property_name = "lsan-allowed"
+
+    def get_value(self, result):
+        # If we have an allowed_match that matched, return None
+        # This value is ignored later (because it matches the default)
+        # We do that because then if we allow a failure in foo/__dir__.ini
+        # we don't want to update foo/bar/__dir__.ini with the same rule
+        if result[1]:
+            return None
+        # Otherwise return the topmost stack frame
+        # TODO: there is probably some improvement to be made by looking for a "better" stack frame
+        return result[0][0]
+
+
+class LeakObjectUpdate(AppendOnlyListUpdate):
+    property_name = "leak-allowed"
+
+    def get_value(self, result):
+        # If we have an allowed_match that matched, return None
+        if result[1]:
+            return None
+        # Otherwise return the process/object name
+        return result[0]
+
+
+class LeakThresholdUpdate(PropertyUpdate):
+    property_name = "leak-threshold"
+    cls_default_value = []
+
+    def __init__(self, node):
+        PropertyUpdate.__init__(self, node)
+        self.thresholds = {}
+
+    def get_value(self, value):
+        threshold = value[2]
+        key = value[0]
+        self.thresholds[key] = threshold
+        return value[:2]
+
+    def value_type(self, data):
+        if all(isinstance(item, tuple) for item in data):
+            return data
+        values = [item.rsplit(":", 1) for item in data]
+        return [(key, int(float(value))) for key, value in values]
+
+    def update_value(self, old_value, new_value, allow_buffer=True):
+        rv = []
+        old_value = dict(old_value)
+        new_value = dict(self.value_type(new_value))
+        for key in set(new_value.keys()) | set(old_value.keys()):
+            old = old_value.get(key, 0)
+            new = new_value.get(key, 0)
+            threshold = self.thresholds.get(key, 0)
+            # If the value is less than the threshold but there isn't
+            # an old value we must have inherited the threshold from
+            # a parent ini file so don't any anything to this one
+            if not old and new < threshold:
+                continue
+            if old >= new:
+                updated = old
+            else:
+                if allow_buffer:
+                    # Round up to nearest 50 kb
+                    boundary = 50 * 1024
+                    updated = int(boundary * ceil(float(new) / boundary))
+                else:
+                    updated = new
+            rv.append((key, updated))
+        return ["%s:%s" % item for item in sorted(rv)]
+
+    def update_default(self):
+        # Current values
+        current_default = []
+        if self.property_name in self.node._data:
+            current_default = [item for item in
+                               self.node._data[self.property_name]
+                               if item.condition_node is None]
+            current_default = current_default[0].value_as(self.value_type)
+        max_new = {}
+        for item in self.new:
+            key, value = item.value
+            if value > max_new.get(key, 0):
+                max_new[key] = value
+        new_value = self.update_value(current_default,
+                                      max_new.items(),
+                                      allow_buffer=False)
+        return True, new_value
+
+
 def group_conditionals(values, property_order=None, boolean_properties=None):
     """Given a list of Value objects, return a list of
     (conditional_node, status) pairs representing the conditional
     expressions that are required to match each status
 
     :param values: List of Values
     :param property_order: List of properties to use in expectation metadata
                            from most to least significant.
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
@@ -284,17 +284,19 @@ class ExpectedUpdater(object):
         self.id_test_map = id_test_map
         self.ignore_existing = ignore_existing
         self.run_info = None
         self.action_map = {"suite_start": self.suite_start,
                            "test_start": self.test_start,
                            "test_status": self.test_status,
                            "test_end": self.test_end,
                            "assertion_count": self.assertion_count,
-                           "lsan_leak": self.lsan_leak}
+                           "lsan_leak": self.lsan_leak,
+                           "mozleak_object": self.mozleak_object,
+                           "mozleak_total": self.mozleak_total}
         self.tests_visited = {}
 
     def update_from_log(self, log_file):
         self.run_info = None
         try:
             data = json.load(log_file)
         except Exception:
             pass
@@ -335,16 +337,25 @@ class ExpectedUpdater(object):
                 asserts = test["asserts"]
                 action_map["assertion_count"]({"test": test["test"],
                                                "count": asserts["count"],
                                                "min_expected": asserts["min"],
                                                "max_expected": asserts["max"]})
         for item in data.get("lsan_leaks", []):
             action_map["lsan_leak"](item)
 
+        mozleak_data = data.get("mozleak", {})
+        for scope, scope_data in mozleak_data.iteritems():
+            for key, action in [("objects", "mozleak_object"),
+                                ("total", "mozleak_total")]:
+                for item in scope_data.get(key, []):
+                    item_data = {"scope": scope}
+                    item_data.update(item)
+                    action_map[action](item_data)
+
     def suite_start(self, data):
         self.run_info = run_info_intern.store(data["run_info"])
 
     def test_start(self, data):
         test_id = intern(data["test"].encode("utf8"))
         try:
             test_data = self.id_test_map[test_id]
         except KeyError:
@@ -392,27 +403,46 @@ class ExpectedUpdater(object):
         test_data = self.id_test_map.get(test_id)
         if test_data is None:
             return
 
         test_data.set(test_id, None, "asserts", self.run_info, data["count"])
         if data["count"] < data["min_expected"] or data["count"] > data["max_expected"]:
             test_data.set_requires_update()
 
-    def lsan_leak(self, data):
+    def test_for_scope(self, data):
         dir_path = data.get("scope", "/")
         dir_id = intern(os.path.join(dir_path, "__dir__").replace(os.path.sep, "/").encode("utf8"))
         if dir_id.startswith("/"):
             dir_id = dir_id[1:]
-        test_data = self.id_test_map[dir_id]
+        return dir_id, self.id_test_map[dir_id]
+
+    def lsan_leak(self, data):
+        dir_id, test_data = self.test_for_scope(data)
         test_data.set(dir_id, None, "lsan",
                       self.run_info, (data["frames"], data.get("allowed_match")))
         if not data.get("allowed_match"):
             test_data.set_requires_update()
 
+    def mozleak_object(self, data):
+        dir_id, test_data = self.test_for_scope(data)
+        test_data.set(dir_id, None, "leak-object",
+                      self.run_info, ("%s:%s", (data["process"], data["name"]),
+                                      data.get("allowed")))
+        if not data.get("allowed"):
+            test_data.set_requires_update()
+
+    def mozleak_total(self, data):
+        if data["bytes"]:
+            dir_id, test_data = self.test_for_scope(data)
+            test_data.set(dir_id, None, "leak-threshold",
+                          self.run_info, (data["process"], data["bytes"], data["threshold"]))
+            if data["bytes"] > data["threshold"] or data["bytes"] < 0:
+                test_data.set_requires_update()
+
 
 def create_test_tree(metadata_path, test_manifest):
     """Create a map of test_id to TestFileData for that test.
     """
     id_test_map = {}
     exclude_types = frozenset(["stub", "helper", "manual", "support", "conformancechecker"])
     all_types = manifestitem.item_types.keys()
     include_types = set(all_types) - exclude_types
@@ -546,16 +576,20 @@ class TestFileData(object):
 
         for test_id, test_data in self.data.iteritems():
             for subtest_id, results_list in test_data.iteritems():
                 for prop, run_info, value in results_list:
                     # Special case directory metadata
                     if subtest_id is None and test_id.endswith("__dir__"):
                         if prop == "lsan":
                             expected.set_lsan(run_info, value)
+                        elif prop == "leak-object":
+                            expected.set_leak_object(run_info, value)
+                        elif prop == "leak-threshold":
+                            expected.set_leak_threshold(run_info, value)
                         continue
 
                     if prop == "status":
                         value = Result(value, default_expected_by_type[self.item_type,
                                                                        subtest_id is not None])
 
                     test_expected = expected_by_test[test_id]
                     if subtest_id is None:
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_update.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_update.py
@@ -48,32 +48,32 @@ def update(tests, *logs):
                                         ["debug", "os", "version", "processor", "bits"],
                                         ["debug"],
                                         False))
 
 
 def create_updater(tests, url_base="/", **kwargs):
     id_test_map = {}
     m = create_test_manifest(tests, url_base)
-    test_manifests = {
-        m: {"url_base": "/",
-            "tests_path": "."}
-    }
+    expected_data = {}
+    metadata.load_expected = lambda _, __, test_path, *args: expected_data[test_path]
+
     for test_path, test_ids, test_type, manifest_str in tests:
         tests = list(m.iterpath(test_path))
         if isinstance(test_ids, (str, unicode)):
             test_ids = [test_ids]
-        test_data = metadata.TestFileData(m, None, test_path, tests)
-        test_data._expected = manifestupdate.compile(BytesIO(manifest_str),
-                                                     test_path,
-                                                     url_base)
+        test_data = metadata.TestFileData("/", "testharness", None, test_path, tests)
+        expected_data[test_path] = manifestupdate.compile(BytesIO(manifest_str),
+                                                          test_path,
+                                                          url_base)
+
         for test_id in test_ids:
             id_test_map[test_id] = test_data
 
-    return id_test_map, metadata.ExpectedUpdater(test_manifests, id_test_map, **kwargs)
+    return id_test_map, metadata.ExpectedUpdater(id_test_map, **kwargs)
 
 
 def create_log(entries):
     data = BytesIO()
     if isinstance(entries, list):
         logger = structuredlog.StructuredLogger("expected_test")
         handler = handlers.StreamHandler(data, formatters.JSONFormatter())
         logger.add_handler(handler)
@@ -602,8 +602,103 @@ def test_update_wptreport_1():
            "results": [],
            "lsan_leaks": [{"scope": "path/to/",
                            "frames": ["baz", "foobar"]}]}
 
     updated = update(tests, log)
 
     assert len(updated) == 1
     assert updated[0][1].get("lsan-allowed") == ["baz"]
+
+
+def test_update_leak_total_0():
+    test_id = "/path/to/test.htm"
+    dir_id = "path/to/__dir__"
+    tests = [("path/to/test.htm", [test_id], "testharness", ""),
+             ("path/to/__dir__", [dir_id], None, "")]
+
+    log_0 = suite_log([("mozleak_total", {"scope": "path/to/",
+                                          "process": "default",
+                                          "bytes": 100,
+                                          "threshold": 0,
+                                          "objects": []})])
+
+    updated = update(tests, log_0)
+    new_manifest = updated[0][1]
+
+    assert not new_manifest.is_empty
+    assert new_manifest.get("leak-threshold") == ['default:110']
+
+
+def test_update_leak_total_1():
+    test_id = "/path/to/test.htm"
+    dir_id = "path/to/__dir__"
+    tests = [("path/to/test.htm", [test_id], "testharness", ""),
+             ("path/to/__dir__", [dir_id], None, "")]
+
+    log_0 = suite_log([("mozleak_total", {"scope": "path/to/",
+                                          "process": "default",
+                                          "bytes": 100,
+                                          "threshold": 1000,
+                                          "objects": []})])
+
+    updated = update(tests, log_0)
+    assert not updated
+
+
+def test_update_leak_total_2():
+    test_id = "/path/to/test.htm"
+    dir_id = "path/to/__dir__"
+    tests = [("path/to/test.htm", [test_id], "testharness", ""),
+             ("path/to/__dir__", [dir_id], None, """
+leak-total: 110""")]
+
+    log_0 = suite_log([("mozleak_total", {"scope": "path/to/",
+                                          "process": "default",
+                                          "bytes": 100,
+                                          "threshold": 110,
+                                          "objects": []})])
+
+    updated = update(tests, log_0)
+    assert not updated
+
+
+def test_update_leak_total_3():
+    test_id = "/path/to/test.htm"
+    dir_id = "path/to/__dir__"
+    tests = [("path/to/test.htm", [test_id], "testharness", ""),
+             ("path/to/__dir__", [dir_id], None, """
+leak-total: 100""")]
+
+    log_0 = suite_log([("mozleak_total", {"scope": "path/to/",
+                                          "process": "default",
+                                          "bytes": 1000,
+                                          "threshold": 100,
+                                          "objects": []})])
+
+    updated = update(tests, log_0)
+    new_manifest = updated[0][1]
+
+    assert not new_manifest.is_empty
+    assert new_manifest.get("leak-threshold") == ['default:1100']
+
+
+def test_update_leak_total_4():
+    test_id = "/path/to/test.htm"
+    dir_id = "path/to/__dir__"
+    tests = [("path/to/test.htm", [test_id], "testharness", ""),
+             ("path/to/__dir__", [dir_id], None, """
+leak-total: 110""")]
+
+    log_0 = suite_log([
+        ("lsan_leak", {"scope": "path/to/",
+                       "frames": ["foo", "bar"]}),
+        ("mozleak_total", {"scope": "path/to/",
+                           "process": "default",
+                           "bytes": 100,
+                           "threshold": 110,
+                           "objects": []})])
+
+    updated = update(tests, log_0)
+    new_manifest = updated[0][1]
+
+    assert not new_manifest.is_empty
+    assert new_manifest.has_key("leak-threshold") is False
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py
@@ -238,16 +238,36 @@ class Test(object):
     def lsan_max_stack_depth(self):
         for meta in self.itermeta(None):
             depth = meta.lsan_max_stack_depth
             if depth is not None:
                 return depth
         return None
 
     @property
+    def mozleak_allowed(self):
+        mozleak_allowed = set()
+        for meta in self.itermeta():
+            mozleak_allowed |= meta.leak_allowed
+            if atom_reset in mozleak_allowed:
+                mozleak_allowed.remove(atom_reset)
+                break
+        return mozleak_allowed
+
+    @property
+    def mozleak_threshold(self):
+        rv = {}
+        for meta in self.itermeta(None):
+            threshold = meta.leak_threshold
+            for key, value in threshold.iteritems():
+                if key not in rv:
+                    rv[key] = value
+        return rv
+
+    @property
     def tags(self):
         tags = set()
         for meta in self.itermeta():
             meta_tags = meta.tags
             tags |= meta_tags
             if atom_reset in meta_tags:
                 tags.remove(atom_reset)
                 break