Bug 1601414 - Add application name and version in vismet perfherder results r=perftest-reviewers,stephendonner
authorRob Wood <rwood@mozilla.com>
Thu, 02 Jan 2020 20:09:45 +0000
changeset 508643 6865e8d68595de0a4ea3edc5f6ea6d5ea872aba8
parent 508642 484bdba6d2ed5ceca3c874fd0dd171e481aa438b
child 508644 4bc6c728cfface4f1da5cabca71c863eb75484a8
push id36974
push userncsoregi@mozilla.com
push dateFri, 03 Jan 2020 03:58:21 +0000
treeherdermozilla-central@4d34e06619ed [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, stephendonner
bugs1601414
milestone73.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1601414 - Add application name and version in vismet perfherder results r=perftest-reviewers,stephendonner Differential Revision: https://phabricator.services.mozilla.com/D58459
taskcluster/docker/visual-metrics/run-visual-metrics.py
--- a/taskcluster/docker/visual-metrics/run-visual-metrics.py
+++ b/taskcluster/docker/visual-metrics/run-visual-metrics.py
@@ -67,16 +67,26 @@ JOB_SCHEMA = Schema(
                 Required("test_name"): str,
                 Required("json_location"): str,
                 Required("video_location"): str,
             }
         ]
     }
 )
 
+#: The schema for validating application data.
+APP_SCHEMA = Schema(
+    {
+        Required("application"): {
+            Required("name"): str,
+            Required("version"): str,
+        }
+    }
+)
+
 PERFHERDER_SCHEMA = Path("/", "builds", "worker", "performance-artifact-schema.json")
 with PERFHERDER_SCHEMA.open() as f:
     PERFHERDER_SCHEMA = json.loads(f.read())
 
 
 def run_command(log, cmd):
     """Run a command using subprocess.check_output
 
@@ -92,36 +102,40 @@ def run_command(log, cmd):
         res = subprocess.check_output(cmd)
         log.info("Command succeeded", result=res)
         return 0, res
     except subprocess.CalledProcessError as e:
         log.info("Command failed", cmd=cmd, status=e.returncode, output=e.output)
         return e.returncode, e.output
 
 
-def append_result(log, suites, test_name, name, result):
+def append_result(log, suites, test_name, app_name, name, result):
     """Appends a ``name`` metrics result in the ``test_name`` suite.
 
     Args:
         log: The structlog logger instance.
         suites: A mapping containing the suites.
         test_name: The name of the test.
+        app_name: The name of the browser application used in the test.
         name: The name of the metrics.
         result: The value to append.
     """
     if name.endswith("Progress"):
         return
     try:
         result = int(result)
     except ValueError:
         log.error("Could not convert value", name=name)
         log.error("%s" % result)
         result = 0
     if test_name not in suites:
-        suites[test_name] = {"name": test_name, "subtests": {}}
+        # TODO: Once Bug 1593198 lands, perfherder will recognize the 'application'; then
+        # it won't be necessary anymore to have the application name added to 'extraOptions'.
+        # So when Bug 1593198 lands, remove the 'extraOptions' key below.
+        suites[test_name] = {"name": test_name, "subtests": {}, "extraOptions": [app_name]}
 
     subtests = suites[test_name]["subtests"]
     if name not in subtests:
         subtests[name] = {
             "name": name,
             "replicates": [result],
             "lowerIsBetter": True,
             "unit": "ms",
@@ -157,16 +171,44 @@ def get_suite(suite):
         The suite.
     """
     suite["subtests"] = [
         compute_median(subtest) for subtest in suite["subtests"].values()
     ]
     return suite
 
 
+def read_json(json_path, schema):
+    """Read the given json file and verify against the provided schema.
+
+    Args:
+        json_path: Filename and path of json file to parse.
+        schema: Schema to validate the json file against.
+
+    Returns:
+        The read json content (dictionary).
+    """
+    try:
+        with open(str(json_path), "r") as f:
+            read_json = json.load(f)
+    except Exception as e:
+        log.error(
+            "Could not read json file: %s" % e, path=json_path, exc_info=True
+        )
+        return 1
+    log.info("Loaded json from file", path=json_path, read_json=read_json)
+
+    try:
+        schema(read_json)
+    except Exception as e:
+        log.error("Failed to parse json: %s" % e)
+        return 1
+    return read_json
+
+
 def main(log, args):
     """Run visualmetrics.py in parallel.
 
     Args:
         log: The structlog logger instance.
         args: The parsed arguments from the argument parser.
 
     Returns:
@@ -191,32 +233,22 @@ def main(log, args):
     except Exception:
         log.error(
             "Could not read extract browsertime results archive",
             path=args.browsertime_results,
             exc_info=True,
         )
         return 1
     log.info("Extracted browsertime results", path=args.browsertime_results)
+
     jobs_json_path = results_path / "browsertime-results" / "jobs.json"
-    try:
-        with open(str(jobs_json_path), "r") as f:
-            jobs_json = json.load(f)
-    except Exception as e:
-        log.error(
-            "Could not read jobs.json file: %s" % e, path=jobs_json_path, exc_info=True
-        )
-        return 1
+    jobs_json = read_json(jobs_json_path, JOB_SCHEMA)
 
-    log.info("Loaded jobs.json from file", path=jobs_json_path, jobs_json=jobs_json)
-    try:
-        JOB_SCHEMA(jobs_json)
-    except Exception as e:
-        log.error("Failed to parse jobs.json: %s" % e)
-        return 1
+    app_json_path = results_path / "browsertime-results" / "application.json"
+    app_json = read_json(app_json_path, APP_SCHEMA)
 
     try:
         downloaded_jobs, failed_downloads = download_inputs(log, jobs_json["jobs"])
     except Exception as e:
         log.error("Failed to download jobs: %s" % e, exc_info=True)
         return 1
 
     failed_runs = 0
@@ -241,21 +273,24 @@ def main(log, args):
                     video_location=job.video_location,
                     error=res,
                 )
                 failed_runs += 1
             else:
                 # Python 3.5 requires a str object (not 3.6+)
                 res = json.loads(res.decode("utf8"))
                 for name, value in res.items():
-                    append_result(log, suites, job.test_name, name, value)
+                    append_result(log, suites, job.test_name, app_json["application"]["name"],
+                                  name, value)
 
     suites = [get_suite(suite) for suite in suites.values()]
+
     perf_data = {
         "framework": {"name": "browsertime"},
+        "application": app_json["application"],
         "type": "vismet",
         "suites": suites,
     }
 
     # Validates the perf data complies with perfherder schema.
     # The perfherder schema uses jsonschema so we can't use voluptuous here.
     validate(perf_data, PERFHERDER_SCHEMA)