Bug 1663417 - make scriptinfo compatible with xpcshell tests r=sparky
authorTarek Ziadé <tarek@mozilla.com>
Thu, 10 Sep 2020 23:15:09 +0000
changeset 548277 667c051572bd9f714c943064e4b2d543a605a302
parent 548276 88fca1e48f23ffc9dbc836ade695d7f950566ee5
child 548278 fd8de30746b91ac76619c569b4a944776feab830
push id37776
push userbtara@mozilla.com
push dateFri, 11 Sep 2020 15:10:42 +0000
treeherdermozilla-central@b133e2d673e8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssparky
bugs1663417
milestone82.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1663417 - make scriptinfo compatible with xpcshell tests r=sparky Differential Revision: https://phabricator.services.mozilla.com/D89372
netwerk/test/unit/test_http3_perf.js
python/mozperftest/mozperftest/runner.py
python/mozperftest/mozperftest/script.py
python/mozperftest/mozperftest/test/browsertime/runner.py
python/mozperftest/mozperftest/test/browsertime/script.py
python/mozperftest/mozperftest/tests/data/samples/test_xpcshell.js
python/mozperftest/mozperftest/tests/data/samples/test_xpcshell_flavor2.js
python/mozperftest/mozperftest/tests/support.py
python/mozperftest/mozperftest/tests/test_script.py
python/mozperftest/setup.py
tools/lint/perfdocs/framework_gatherers.py
--- a/netwerk/test/unit/test_http3_perf.js
+++ b/netwerk/test/unit/test_http3_perf.js
@@ -1,10 +1,17 @@
 "use strict";
 
+var perfMetadata = {
+  owner: "Network Team",
+  name: "http3 raw",
+  description:
+    "XPCShell tests that verifies the lib integration against a local server",
+};
+
 var performance = performance || {};
 performance.now = (function() {
   return (
     performance.now ||
     performance.mozNow ||
     performance.msNow ||
     performance.oNow ||
     performance.webkitNow ||
--- a/python/mozperftest/mozperftest/runner.py
+++ b/python/mozperftest/mozperftest/runner.py
@@ -128,17 +128,17 @@ def run_tests(mach_cmd, **kwargs):
             flavor = kwargs["flavor"]
             kwargs["tests"], tmp_dir = build_test_list(
                 kwargs["tests"], randomized=flavor != "doc"
             )
             try:
                 # XXX this doc is specific to browsertime scripts
                 # maybe we want to move it
                 if flavor == "doc":
-                    from mozperftest.test.browsertime.script import ScriptInfo
+                    from mozperftest.script import ScriptInfo
 
                     for test in kwargs["tests"]:
                         print(ScriptInfo(test))
                     return
 
                 env = MachEnvironment(mach_cmd, hooks=hooks, **kwargs)
                 metadata = Metadata(mach_cmd, env, flavor)
                 hooks.run("before_runs", env)
rename from python/mozperftest/mozperftest/test/browsertime/script.py
rename to python/mozperftest/mozperftest/script.py
--- a/python/mozperftest/mozperftest/test/browsertime/script.py
+++ b/python/mozperftest/mozperftest/script.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from collections import defaultdict
 import re
 import textwrap
 from pathlib import Path
+from enum import Enum
 
 import esprima
 
 
 # list of metadata, each item is the name and if the field is mandatory
 METADATA = [
     ("setUp", False),
     ("tearDown", False),
@@ -37,67 +38,111 @@ Test Name: %(name)s
 Usage:
 %(usage)s
 
 Description:
 %(longDescription)s
 """
 
 
+XPCSHELL_FUNCS = "add_task", "run_test", "run_next_test"
+
+
 class MissingFieldError(Exception):
     pass
 
 
+class ScriptType(Enum):
+    XPCSHELL = 1
+    BROWSERTIME = 2
+
+
 class ScriptInfo(defaultdict):
     """Loads and parses a Browsertime test script."""
 
     def __init__(self, path):
         super(ScriptInfo, self).__init__()
         self.script = Path(path)
         self["filename"] = str(self.script)
+        self.script_type = ScriptType.BROWSERTIME
 
         with self.script.open() as f:
             self.parsed = esprima.parseScript(f.read())
 
         # looking for the exports statement
         for stmt in self.parsed.body:
+            #  detecting if the script has add_task()
+            if (
+                stmt.type == "ExpressionStatement"
+                and stmt.expression is not None
+                and stmt.expression.callee is not None
+                and stmt.expression.callee.type == "Identifier"
+                and stmt.expression.callee.name in XPCSHELL_FUNCS
+            ):
+                self["test"] = "xpcshell"
+                self.script_type = ScriptType.XPCSHELL
+                continue
+
+            # plain xpcshell tests functions markers
+            if stmt.type == "FunctionDeclaration" and stmt.id.name in XPCSHELL_FUNCS:
+                self["test"] = "xpcshell"
+                self.script_type = ScriptType.XPCSHELL
+                continue
+
+            # is this the perfMetdatata plain var ?
+            if stmt.type == "VariableDeclaration":
+                for decl in stmt.declarations:
+                    if (
+                        decl.type != "VariableDeclarator"
+                        or decl.id.type != "Identifier"
+                        or decl.id.name != "perfMetadata"
+                        or decl.init is None
+                    ):
+                        continue
+                    self.scan_properties(decl.init.properties)
+                    continue
+
+            # or the module.exports map ?
             if (
                 stmt.type != "ExpressionStatement"
                 or stmt.expression.left is None
                 or stmt.expression.left.property is None
                 or stmt.expression.left.property.name != "exports"
                 or stmt.expression.right is None
                 or stmt.expression.right.properties is None
             ):
                 continue
 
             # now scanning the properties
-            for prop in stmt.expression.right.properties:
-                if prop.value.type == "Identifier":
-                    value = prop.value.name
-                elif prop.value.type == "Literal":
-                    value = prop.value.value
-                elif prop.value.type == "TemplateLiteral":
-                    # ugly
-                    value = prop.value.quasis[0].value.cooked.replace("\n", " ")
-                    value = re.sub(r"\s+", " ", value).strip()
-                elif prop.value.type == "ArrayExpression":
-                    value = [e.value for e in prop.value.elements]
-                else:
-                    raise ValueError(prop.value.type)
-
-                self[prop.key.name] = value
+            self.scan_properties(stmt.expression.right.properties)
 
         # If the fields found, don't match our known ones, then an error is raised
         for field, required in METADATA:
             if not required:
                 continue
             if field not in self:
                 raise MissingFieldError(field)
 
+    def scan_properties(self, properties):
+        for prop in properties:
+            if prop.value.type == "Identifier":
+                value = prop.value.name
+            elif prop.value.type == "Literal":
+                value = prop.value.value
+            elif prop.value.type == "TemplateLiteral":
+                # ugly
+                value = prop.value.quasis[0].value.cooked.replace("\n", " ")
+                value = re.sub(r"\s+", " ", value).strip()
+            elif prop.value.type == "ArrayExpression":
+                value = [e.value for e in prop.value.elements]
+            else:
+                raise ValueError(prop.value.type)
+
+            self[prop.key.name] = value
+
     def __str__(self):
         """Used to generate docs."""
         d = defaultdict(lambda: "N/A")
         for field, value in self.items():
             if field == "filename":
                 d[field] = self.script.name
                 continue
 
--- a/python/mozperftest/mozperftest/test/browsertime/runner.py
+++ b/python/mozperftest/mozperftest/test/browsertime/runner.py
@@ -7,17 +7,17 @@ import os
 import pathlib
 import sys
 import re
 import shutil
 from pathlib import Path
 
 from mozperftest.utils import install_package
 from mozperftest.test.noderunner import NodeRunner
-from mozperftest.test.browsertime.script import ScriptInfo
+from mozperftest.script import ScriptInfo
 
 
 BROWSERTIME_SRC_ROOT = Path(__file__).parent
 PILLOW_VERSION = "7.2.0"
 PYSSIM_VERSION = "0.4"
 
 
 def matches(args, *flags):
--- a/python/mozperftest/mozperftest/tests/data/samples/test_xpcshell.js
+++ b/python/mozperftest/mozperftest/tests/data/samples/test_xpcshell.js
@@ -9,8 +9,23 @@ add_task(async function dummy_test() {
   /*
    * Do some test here, get some metrics
    */
   var metrics = {"metrics1": 1, "metrics2": 2};
   info("perfMetrics", metrics);
   info("perfMetrics", {"metrics3": 3});
   await true;
 });
+
+
+var perfMetadata = {
+  owner: "Performance Testing Team",
+  name: "Example",
+  description: "The description of the example test.",
+  longDescription: `
+  This is a longer description of the test perhaps including information
+  about how it should be run locally or links to relevant information.
+  `,
+  usage: "explains how to use it",
+  supportedBrowsers: ["Firefox"],
+  supportedPlatforms: ["Desktop"],
+};
+
new file mode 100644
--- /dev/null
+++ b/python/mozperftest/mozperftest/tests/data/samples/test_xpcshell_flavor2.js
@@ -0,0 +1,29 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var perfMetadata = {
+  owner: "Performance Testing Team",
+  name: "Example",
+  description: "The description of the example test.",
+  longDescription: `
+  This is a longer description of the test perhaps including information
+  about how it should be run locally or links to relevant information.
+  `,
+  usage: `
+  ./mach perftest
+    python/mozperftest/mozperftest/tests/data/samples/test_xpcshell_flavor2.js
+  `,
+  supportedBrowsers: ["Firefox"],
+  supportedPlatforms: ["Desktop"],
+};
+
+function run_next_test() {
+  // do something
+}
+
+function run_test() {
+  // do something
+}
--- a/python/mozperftest/mozperftest/tests/support.py
+++ b/python/mozperftest/mozperftest/tests/support.py
@@ -9,16 +9,17 @@ from mozperftest.metadata import Metadat
 from mozperftest.environment import MachEnvironment
 from mozperftest.hooks import Hooks
 
 
 HERE = Path(__file__).parent
 EXAMPLE_TESTS_DIR = os.path.join(HERE, "data", "samples")
 EXAMPLE_TEST = os.path.join(EXAMPLE_TESTS_DIR, "perftest_example.js")
 EXAMPLE_XPCSHELL_TEST = Path(EXAMPLE_TESTS_DIR, "test_xpcshell.js")
+EXAMPLE_XPCSHELL_TEST2 = Path(EXAMPLE_TESTS_DIR, "test_xpcshell_flavor2.js")
 BT_DATA = Path(HERE, "data", "browsertime-results", "browsertime.json")
 BT_DATA_VIDEO = Path(HERE, "data", "browsertime-results-video", "browsertime.json")
 DMG = Path(HERE, "data", "firefox.dmg")
 MOZINFO = Path(HERE, "data", "mozinfo.json")
 
 
 @contextlib.contextmanager
 def temp_file(name="temp", content=None):
--- a/python/mozperftest/mozperftest/tests/test_script.py
+++ b/python/mozperftest/mozperftest/tests/test_script.py
@@ -1,25 +1,41 @@
 #!/usr/bin/env python
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import mozunit
 import pytest
 
-from mozperftest.test.browsertime.script import ScriptInfo, MissingFieldError
-from mozperftest.tests.support import EXAMPLE_TEST, HERE
+from mozperftest.script import ScriptInfo, MissingFieldError, ScriptType
+from mozperftest.tests.support import (
+    EXAMPLE_TEST,
+    HERE,
+    EXAMPLE_XPCSHELL_TEST,
+    EXAMPLE_XPCSHELL_TEST2,
+)
 
 
-def test_scriptinfo():
+def test_scriptinfo_bt():
     info = ScriptInfo(EXAMPLE_TEST)
     assert info["author"] == "N/A"
 
     display = str(info)
     assert "The description of the example test." in display
+    assert info.script_type == ScriptType.BROWSERTIME
+
+
+@pytest.mark.parametrize("script", [EXAMPLE_XPCSHELL_TEST, EXAMPLE_XPCSHELL_TEST2])
+def test_scriptinfo_xpcshell(script):
+    info = ScriptInfo(script)
+    assert info["author"] == "N/A"
+
+    display = str(info)
+    assert "The description of the example test." in display
+    assert info.script_type == ScriptType.XPCSHELL
 
 
 def test_scriptinfo_failure():
     bad_example = HERE / "data" / "failing-samples" / "perftest_doc_failure_example.js"
     with pytest.raises(MissingFieldError):
         ScriptInfo(bad_example)
 
 
--- a/python/mozperftest/setup.py
+++ b/python/mozperftest/setup.py
@@ -4,18 +4,25 @@
 
 from __future__ import absolute_import
 
 from setuptools import setup
 
 PACKAGE_NAME = "mozperftest"
 PACKAGE_VERSION = "0.2"
 
-deps = ["regex", "jsonschema", "mozlog >= 6.0", "mozdevice >= 4.0.0", "mozproxy",
-        "mozinfo", "mozfile"]
+deps = [
+    "regex",
+    "jsonschema",
+    "mozlog >= 6.0",
+    "mozdevice >= 4.0.0",
+    "mozproxy",
+    "mozinfo",
+    "mozfile",
+]
 
 setup(
     name=PACKAGE_NAME,
     version=PACKAGE_VERSION,
     description="Mozilla's mach perftest command",
     classifiers=["Programming Language :: Python :: 3.6"],
     # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
     keywords="",
--- a/tools/lint/perfdocs/framework_gatherers.py
+++ b/tools/lint/perfdocs/framework_gatherers.py
@@ -5,17 +5,17 @@ from __future__ import absolute_import
 
 import collections
 import os
 import pathlib
 import re
 
 from perfdocs.utils import read_yaml
 from manifestparser import TestManifest
-from mozperftest.test.browsertime.script import ScriptInfo
+from mozperftest.script import ScriptInfo
 
 """
 This file is for framework specific gatherers since manifests
 might be parsed differently in each of them. The gatherers
 must implement the FrameworkGatherer class.
 """