testing/mozharness/external_tools/performance-artifact-schema.json
author Sandor Molnar <smolnar@mozilla.com>
Fri, 24 Sep 2021 00:43:42 +0300
changeset 593102 4eda9eb8926bdd50f4b80128ce3475eb7c6d9a4d
parent 559498 403a7e5ec5fc69ac90533b75c059d19b79be1832
permissions -rw-r--r--
Merge autoland to mozilla-central. a=merge

{
    "definitions": {
        "application_schema": {
            "properties": {
                "name": {
                    "title": "Application under performance test",
                    "enum": [
                        "firefox",
                        "chrome",
                        "chrome-m",
                        "chromium",
                        "fennec",
                        "geckoview",
                        "refbrow",
                        "fenix"
                    ],
                    "maxLength": 10,
                    "type": "string"
                },
                "version": {
                    "title": "Application's version",
                    "maxLength": 40,
                    "type": "string"
                }
            },
            "required": ["name"],
            "type": "object"
        },
        "framework_schema": {
            "properties": {
                "name": {
                    "title": "Framework name",
                    "type": "string"
                }
            },
            "type": "object"
        },
        "subtest_schema": {
            "properties": {
                "name": {
                    "title": "Subtest name",
                    "type": "string"
                },
                "publicName": {
                    "title": "Public subtest name",
                    "description": "Allows renaming test's name, without breaking existing performance data series",
                    "maxLength": 30,
                    "type": "string"
                },
                "value": {
                    "description": "Summary value for subtest",
                    "title": "Subtest value",
                    "type": "number",
                    "minimum": -1000000000000.0,
                    "maximum": 1000000000000.0
                },
                "unit": {
                    "title": "Measurement unit",
                    "type": "string",
                    "minLength": 1,
                    "maxLength": 20
                },
                "lowerIsBetter": {
                    "description": "Whether lower values are better for subtest",
                    "title": "Lower is better",
                    "type": "boolean"
                },
                "shouldAlert": {
                    "description": "Whether we should alert",
                    "title": "Should alert",
                    "type": "boolean"
                },
                "alertThreshold": {
                    "description": "% change threshold before alerting",
                    "title": "Alert threshold",
                    "type": "number",
                    "minimum": 0.0,
                    "maximum": 1000.0
                },
                "minBackWindow": {
                    "description": "Minimum back window to use for alerting",
                    "title": "Minimum back window",
                    "type": "number",
                    "minimum": 1,
                    "maximum": 255
                },
                "maxBackWindow": {
                    "description": "Maximum back window to use for alerting",
                    "title": "Maximum back window",
                    "type": "number",
                    "minimum": 1,
                    "maximum": 255
                },
                "foreWindow": {
                    "description": "Fore window to use for alerting",
                    "title": "Fore window",
                    "type": "number",
                    "minimum": 1,
                    "maximum": 255
                }
            },
            "required": [
                "name",
                "value"
            ],
            "type": "object"
        },
        "suite_schema": {
            "properties": {
                "name": {
                    "title": "Suite name",
                    "type": "string"
                },
                "publicName": {
                    "title": "Public suite name",
                    "description": "Allows renaming suite's name, without breaking existing performance data series",
                    "maxLength": 30,
                    "type": "string"
                },
                "tags": {
                    "type": "array",
                    "title": "Free form tags, which ease the grouping & searching of performance tests",
                    "description": "Similar to extraOptions, except it does not break existing performance data series",
                    "items": {
                        "type": "string",
                        "pattern": "^[a-zA-Z0-9-]{1,24}$"
                    },
                    "uniqueItems": true,
                    "maxItems": 14
                },
                "extraOptions": {
                    "type": "array",
                    "title": "Extra options used in running suite",
                    "items": {
                        "type": "string",
                        "maxLength": 100
                    },
                    "uniqueItems": true,
                    "maxItems": 8
                },
                "subtests": {
                    "items": {
                        "$ref": "#/definitions/subtest_schema"
                    },
                    "title": "Subtests",
                    "type": "array"
                },
                "value": {
                    "title": "Suite value",
                    "type": "number",
                    "minimum": -1000000000000.0,
                    "maximum": 1000000000000.0
                },
                "unit": {
                    "title": "Measurement unit",
                    "type": "string",
                    "minLength": 1,
                    "maxLength": 20
                },
                "lowerIsBetter": {
                    "description": "Whether lower values are better for suite",
                    "title": "Lower is better",
                    "type": "boolean"
                },
                "shouldAlert": {
                    "description": "Whether we should alert on this suite (overrides default behaviour)",
                    "title": "Should alert",
                    "type": "boolean"
                },
                "alertThreshold": {
                    "description": "% change threshold before alerting",
                    "title": "Alert threshold",
                    "type": "number",
                    "minimum": 0.0,
                    "maximum": 1000.0
                },
                "minBackWindow": {
                    "description": "Minimum back window to use for alerting",
                    "title": "Minimum back window",
                    "type": "integer",
                    "minimum": 1,
                    "maximum": 255
                },
                "maxBackWindow": {
                    "description": "Maximum back window to use for alerting",
                    "title": "Maximum back window",
                    "type": "integer",
                    "minimum": 1,
                    "maximum": 255
                },
                "foreWindow": {
                    "description": "Fore window to use for alerting",
                    "title": "Fore window",
                    "type": "integer",
                    "minimum": 1,
                    "maximum": 255
                }
            },
            "required": [
                "name",
                "subtests"
            ],
            "type": "object"
        }
    },
    "description": "Structure for submitting performance data as part of a job",
    "id": "https://treeherder.mozilla.org/schemas/v1/performance-artifact.json#",
    "properties": {
        "application":{
            "$ref": "#/definitions/application_schema"
        },
        "framework": {
            "$ref": "#/definitions/framework_schema"
        },
        "suites": {
            "description": "List of suite-level data submitted as part of this structure",
            "items": {
                "$ref": "#/definitions/suite_schema"
            },
            "title": "Performance suites",
            "type": "array"
        }
    },
    "required": [
        "framework",
        "suites"
    ],
    "title": "Perfherder Schema",
    "type": "object"
}