testing/mozharness/mozharness/mozilla/testing/errors.py
author Andrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 22 Apr 2019 22:32:34 +0000
changeset 470488 60c512bab3e986b609d7fda44d827c21d90612d2
parent 448369 cd593d522d2486ca982697e4b6c620924894b6fb
child 472053 732976d3f555a01d9032ede89a28121502ee49e2
permissions -rw-r--r--
Bug 1523303 - Align mozharness suite names with the ones in 'moztest.resolve', r=gbrown This officially makes 'moztest.resolve' the source of truth when it comes to suite names. It aligns that file with the names used in both the desktop_unittest and android_emulator_unittest scripts. Differential Revision: https://phabricator.services.mozilla.com/D27555

#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""Mozilla error lists for running tests.

Error lists are used to parse output in mozharness.base.log.OutputParser.

Each line of output is matched against each substring or regular expression
in the error list.  On a match, we determine the 'level' of that line,
whether IGNORE, DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL.

"""

import re
from mozharness.base.log import INFO, WARNING, ERROR

# ErrorLists {{{1
_mochitest_summary = {
    'regex': re.compile(r'''(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))'''),  # NOQA: E501
    'pass_group': "Passed",
    'fail_group': "Failed",
    'known_fail_group': "Todo",
}

TinderBoxPrintRe = {
    "mochitest-chrome_summary": _mochitest_summary,
    "mochitest-webgl1-core_summary": _mochitest_summary,
    "mochitest-webgl1-ext_summary": _mochitest_summary,
    "mochitest-webgl2-core_summary": _mochitest_summary,
    "mochitest-webgl2-ext_summary": _mochitest_summary,
    "mochitest-webgl2-deqp_summary": _mochitest_summary,
    "mochitest-media_summary": _mochitest_summary,
    "mochitest-plain_summary": _mochitest_summary,
    "mochitest-plain-clipboard_summary": _mochitest_summary,
    "mochitest-plain-gpu_summary": _mochitest_summary,
    "marionette_summary": {
        'regex': re.compile(r'''(passed|failed|todo):\ +(\d+)'''),
        'pass_group': "passed",
        'fail_group': "failed",
        'known_fail_group': "todo",
    },
    "reftest_summary": {
        'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
        'pass_group': "Successful",
        'fail_group': "Unexpected",
        'known_fail_group': "Known problems",
    },
    "crashtest_summary": {
        'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
        'pass_group': "Successful",
        'fail_group': "Unexpected",
        'known_fail_group': "Known problems",
    },
    "xpcshell_summary": {
        'regex': re.compile(r'''INFO \| (Passed|Failed|Todo): (\d+)'''),
        'pass_group': "Passed",
        'fail_group': "Failed",
        'known_fail_group': "Todo",
    },
    "jsreftest_summary": {
        'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
        'pass_group': "Successful",
        'fail_group': "Unexpected",
        'known_fail_group': "Known problems",
    },
    "robocop_summary": _mochitest_summary,
    "instrumentation_summary": _mochitest_summary,
    "cppunittest_summary": {
        'regex': re.compile(r'''cppunittests INFO \| (Passed|Failed): (\d+)'''),
        'pass_group': "Passed",
        'fail_group': "Failed",
        'known_fail_group': None,
    },
    "gtest_summary": {
        'regex': re.compile(r'''(Passed|Failed): (\d+)'''),
        'pass_group': "Passed",
        'fail_group': "Failed",
        'known_fail_group': None,
    },
    "jittest_summary": {
        'regex': re.compile(r'''(Passed|Failed): (\d+)'''),
        'pass_group': "Passed",
        'fail_group': "Failed",
        'known_fail_group': None,
    },
    "mozbase_summary": {
        'regex': re.compile(r'''(OK)|(FAILED) \(errors=(\d+)'''),
        'pass_group': "OK",
        'fail_group': "FAILED",
        'known_fail_group': None,
    },
    "mozmill_summary": {
        'regex': re.compile(r'''INFO (Passed|Failed|Skipped): (\d+)'''),
        'pass_group': "Passed",
        'fail_group': "Failed",
        'known_fail_group': "Skipped",
    },
    "geckoview_summary": {
        'regex': re.compile(r'''(Passed|Failed): (\d+)'''),
        'pass_group': "Passed",
        'fail_group': "Failed",
        'known_fail_group': None,
    },
    "geckoview-junit_summary": {
        'regex': re.compile(r'''(Passed|Failed): (\d+)'''),
        'pass_group': "Passed",
        'fail_group': "Failed",
        'known_fail_group': None,
    },

    "harness_error": {
        'full_regex': re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"),  # NOQA: E501
        'minimum_regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH)'''),
        'retry_regex': re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|ADBError|ADBProcessError|ADBTimeoutError|program finished with exit code 80|INFRA-ERROR)''')  # NOQA: E501
    },
}

TestPassed = [
    {'regex': re.compile('''(TEST-INFO|TEST-KNOWN-FAIL|TEST-PASS|INFO \| )'''), 'level': INFO},
]

HarnessErrorList = [
    {'substr': 'TEST-UNEXPECTED', 'level': ERROR, },
    {'substr': 'PROCESS-CRASH', 'level': ERROR, },
    {'regex': re.compile('''ERROR: (Address|Leak)Sanitizer'''), 'level': ERROR, },
    {'regex': re.compile('''thread '([^']+)' panicked'''), 'level': ERROR, },
    {'substr': 'pure virtual method called', 'level': ERROR, },
    {'substr': 'Pure virtual function called!', 'level': ERROR, },
]

LogcatErrorList = [
    {'substr': 'Fatal signal 11 (SIGSEGV)', 'level': ERROR,
     'explanation': 'This usually indicates the B2G process has crashed'},
    {'substr': 'Fatal signal 7 (SIGBUS)', 'level': ERROR,
     'explanation': 'This usually indicates the B2G process has crashed'},
    {'substr': '[JavaScript Error:', 'level': WARNING},
    {'substr': 'seccomp sandbox violation', 'level': ERROR,
        'explanation': 'A content process has violated the system call sandbox (bug 790923)'},
]