deleted file mode 100644
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/stability.py
+++ /dev/null
@@ -1,279 +0,0 @@
-import copy
-import functools
-import imp
-import os
-import sys
-from collections import OrderedDict, defaultdict
-from datetime import datetime
-
-from mozlog import reader
-from mozlog.formatters import JSONFormatter, TbplFormatter
-from mozlog.handlers import BaseHandler, LogLevelFilter, StreamHandler
-
-here = os.path.dirname(__file__)
-localpaths = imp.load_source("localpaths", os.path.abspath(os.path.join(here, os.pardir, os.pardir, "localpaths.py")))
-from wpt.markdown import markdown_adjust, table
-
-
-class LogActionFilter(BaseHandler):
-
- """Handler that filters out messages not of a given set of actions.
-
- Subclasses BaseHandler.
-
- :param inner: Handler to use for messages that pass this filter
- :param actions: List of actions for which to fire the handler
- """
-
- def __init__(self, inner, actions):
- """Extend BaseHandler and set inner and actions props on self."""
- BaseHandler.__init__(self, inner)
- self.inner = inner
- self.actions = actions
-
- def __call__(self, item):
- """Invoke handler if action is in list passed as constructor param."""
- if item["action"] in self.actions:
- return self.inner(item)
-
-
-class LogHandler(reader.LogHandler):
-
- """Handle updating test and subtest status in log.
-
- Subclasses reader.LogHandler.
- """
- def __init__(self):
- self.results = OrderedDict()
-
- def find_or_create_test(self, data):
- test_name = data["test"]
- if self.results.get(test_name):
- return self.results[test_name]
-
- test = {
- "subtests": OrderedDict(),
- "status": defaultdict(int)
- }
- self.results[test_name] = test
- return test
-
- def find_or_create_subtest(self, data):
- test = self.find_or_create_test(data)
- subtest_name = data["subtest"]
-
- if test["subtests"].get(subtest_name):
- return test["subtests"][subtest_name]
-
- subtest = {
- "status": defaultdict(int),
- "messages": set()
- }
- test["subtests"][subtest_name] = subtest
-
- return subtest
-
- def test_status(self, data):
- subtest = self.find_or_create_subtest(data)
- subtest["status"][data["status"]] += 1
- if data.get("message"):
- subtest["messages"].add(data["message"])
-
- def test_end(self, data):
- test = self.find_or_create_test(data)
- test["status"][data["status"]] += 1
-
-
-def is_inconsistent(results_dict, iterations):
- """Return whether or not a single test is inconsistent."""
- return len(results_dict) > 1 or sum(results_dict.values()) != iterations
-
-
-def process_results(log, iterations):
- """Process test log and return overall results and list of inconsistent tests."""
- inconsistent = []
- handler = LogHandler()
- reader.handle_log(reader.read(log), handler)
- results = handler.results
- for test_name, test in results.iteritems():
- if is_inconsistent(test["status"], iterations):
- inconsistent.append((test_name, None, test["status"], []))
- for subtest_name, subtest in test["subtests"].iteritems():
- if is_inconsistent(subtest["status"], iterations):
- inconsistent.append((test_name, subtest_name, subtest["status"], subtest["messages"]))
- return results, inconsistent
-
-
-def err_string(results_dict, iterations):
- """Create and return string with errors from test run."""
- rv = []
- total_results = sum(results_dict.values())
- for key, value in sorted(results_dict.items()):
- rv.append("%s%s" %
- (key, ": %s/%s" % (value, iterations) if value != iterations else ""))
- if total_results < iterations:
- rv.append("MISSING: %s/%s" % (iterations - total_results, iterations))
- rv = ", ".join(rv)
- if is_inconsistent(results_dict, iterations):
- rv = "**%s**" % rv
- return rv
-
-
-def write_inconsistent(log, inconsistent, iterations):
- """Output inconsistent tests to logger.error."""
- log("## Unstable results ##\n")
- strings = [(
- "`%s`" % markdown_adjust(test),
- ("`%s`" % markdown_adjust(subtest)) if subtest else "",
- err_string(results, iterations),
- ("`%s`" % markdown_adjust(";".join(messages))) if len(messages) else "")
- for test, subtest, results, messages in inconsistent]
- table(["Test", "Subtest", "Results", "Messages"], strings, log)
-
-
-def write_results(log, results, iterations, pr_number=None, use_details=False):
- log("## All results ##\n")
- if use_details:
- log("<details>\n")
- log("<summary>%i %s ran</summary>\n\n" % (len(results),
- "tests" if len(results) > 1
- else "test"))
-
- for test_name, test in results.iteritems():
- baseurl = "http://w3c-test.org/submissions"
- if "https" in os.path.splitext(test_name)[0].split(".")[1:]:
- baseurl = "https://w3c-test.org/submissions"
- title = test_name
- if use_details:
- log("<details>\n")
- if pr_number:
- title = "<a href=\"%s/%s%s\">%s</a>" % (baseurl, pr_number, test_name, title)
- log('<summary>%s</summary>\n\n' % title)
- else:
- log("### %s ###" % title)
- strings = [("", err_string(test["status"], iterations), "")]
-
- strings.extend(((
- ("`%s`" % markdown_adjust(subtest_name)) if subtest else "",
- err_string(subtest["status"], iterations),
- ("`%s`" % markdown_adjust(';'.join(subtest["messages"]))) if len(subtest["messages"]) else "")
- for subtest_name, subtest in test["subtests"].items()))
- table(["Subtest", "Results", "Messages"], strings, log)
- if use_details:
- log("</details>\n")
-
- if use_details:
- log("</details>\n")
-
-
-def run_step(logger, iterations, restart_after_iteration, kwargs_extras, **kwargs):
- import wptrunner
- kwargs = copy.deepcopy(kwargs)
-
- if restart_after_iteration:
- kwargs["repeat"] = iterations
- else:
- kwargs["rerun"] = iterations
-
- kwargs["pause_after_test"] = False
- kwargs.update(kwargs_extras)
-
- handler = LogActionFilter(
- LogLevelFilter(
- StreamHandler(
- sys.stdout,
- TbplFormatter()
- ),
- "WARNING"),
- ["log", "process_output"])
-
- # There is a public API for this in the next mozlog
- initial_handlers = logger._state.handlers
- logger._state.handlers = []
-
- with open("raw.log", "wb") as log:
- # Setup logging for wptrunner that keeps process output and
- # warning+ level logs only
- logger.add_handler(handler)
- logger.add_handler(StreamHandler(log, JSONFormatter()))
-
- wptrunner.run_tests(**kwargs)
-
- logger._state.handlers = initial_handlers
-
- with open("raw.log", "rb") as log:
- results, inconsistent = process_results(log, iterations)
- return results, inconsistent, iterations
-
-
-def get_steps(logger, repeat_loop, repeat_restart, kwargs_extras):
- steps = []
- for kwargs_extra in kwargs_extras:
- if kwargs_extra:
- flags_string = " with flags %s" % " ".join(
- "%s=%s" % item for item in kwargs_extra.iteritems())
- else:
- flags_string = ""
-
- if repeat_loop:
- desc = "Running tests in a loop %d times%s" % (repeat_loop,
- flags_string)
- steps.append((desc, functools.partial(run_step, logger, repeat_loop, False, kwargs_extra)))
-
- if repeat_restart:
- desc = "Running tests in a loop with restarts %s times%s" % (repeat_restart,
- flags_string)
- steps.append((desc, functools.partial(run_step, logger, repeat_restart, True, kwargs_extra)))
-
- return steps
-
-
-def write_summary(logger, step_results, final_result):
- for desc, result in step_results:
- logger.info('::: %s : %s' % (desc, result))
- logger.info(':::')
- if final_result == "PASS":
- log = logger.info
- elif final_result == "TIMEOUT":
- log = logger.warning
- else:
- log = logger.error
- log('::: Test verification %s' % final_result)
-
- logger.info(':::')
-
-
-def check_stability(logger, repeat_loop=10, repeat_restart=5, chaos_mode=True, max_time=None,
- output_results=True, **kwargs):
- kwargs_extras = [{}]
- if chaos_mode and kwargs["product"] == "firefox":
- kwargs_extras.append({"chaos_mode_flags": 3})
-
- steps = get_steps(logger, repeat_loop, repeat_restart, kwargs_extras)
-
- start_time = datetime.now()
- step_results = []
-
- for desc, step_func in steps:
- if max_time and datetime.now() - start_time > max_time:
- logger.info("::: Test verification is taking too long: Giving up!")
- logger.info("::: So far, all checks passed, but not all checks were run.")
- write_summary(logger, step_results, "TIMEOUT")
- return 2
-
- logger.info(':::')
- logger.info('::: Running test verification step "%s"...' % desc)
- logger.info(':::')
- results, inconsistent, iterations = step_func(**kwargs)
- if output_results:
- write_results(logger.info, results, iterations)
-
- if inconsistent:
- step_results.append((desc, "FAIL"))
- write_inconsistent(logger.info, inconsistent, iterations)
- write_summary(logger, step_results, "FAIL")
- return 1
-
- step_results.append((desc, "PASS"))
-
- write_summary(logger, step_results, "PASS")