Bug 1480841 - Raptor support for running speedometer on geckoview r=jmaher
authorRob Wood <rwood@mozilla.com>
Fri, 17 Aug 2018 18:06:22 +0000
changeset 487235 90caf8ba542c211a4ed08d4029df6f3d200003c4
parent 487234 d3430f59b182b466c0530966d193f8226f8c4de4
child 487236 2f1bbddc826bb9af360c7c928068caa2a2802f5a
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1480841
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1480841 - Raptor support for running speedometer on geckoview r=jmaher Differential Revision: https://phabricator.services.mozilla.com/D3446
testing/mozharness/mozharness/mozilla/testing/raptor.py
testing/profiles/raptor/user.js
testing/raptor/raptor/cmdline.py
testing/raptor/raptor/control_server.py
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/raptor-speedometer.ini
testing/raptor/requirements.txt
testing/raptor/test/test_cmdline.py
testing/raptor/webext/raptor/benchmark-relay.js
testing/raptor/webext/raptor/runner.js
third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -50,17 +50,17 @@ class Raptor(TestingMixin, MercurialScri
     config_options = [
         [["--test"],
          {"action": "store",
           "dest": "test",
           "help": "Raptor test to run"
           }],
         [["--app"],
          {"default": "firefox",
-          "choices": ["firefox", "chrome"],
+          "choices": ["firefox", "chrome", "geckoview"],
           "dest": "app",
           "help": "name of the application we are testing (default: firefox)"
           }],
         [["--branch-name"],
          {"action": "store",
           "dest": "branch",
           "help": "branch running against"
           }],
@@ -104,22 +104,26 @@ class Raptor(TestingMixin, MercurialScri
         self.run_local = self.config.get('run_local')
 
         # app (browser testing on) defaults to firefox
         self.app = "firefox"
 
         if self.run_local:
             # raptor initiated locally, get app from command line args
             # which are passed in from mach inside 'raptor_cmd_line_args'
+            # cmd line args can be in two formats depending on how user entered them
+            # i.e. "--app=geckoview" or separate as "--app", "geckoview" so we have to
+            # check each cmd line arg individually
             self.app = "firefox"
             if 'raptor_cmd_line_args' in self.config:
-                for next_arg in self.config['raptor_cmd_line_args']:
-                    if "chrome" in next_arg:
-                        self.app = "chrome"
-                        break
+                for app in ['chrome', 'geckoview']:
+                    for next_arg in self.config['raptor_cmd_line_args']:
+                        if app in next_arg:
+                            self.app = app
+                            break
         else:
             # raptor initiated in production via mozharness
             self.test = self.config['test']
             self.app = self.config.get("app", "firefox")
 
         self.installer_url = self.config.get("installer_url")
         self.raptor_json_url = self.config.get("raptor_json_url")
         self.raptor_json = self.config.get("raptor_json")
--- a/testing/profiles/raptor/user.js
+++ b/testing/profiles/raptor/user.js
@@ -1,3 +1,6 @@
 // Preferences file used by the raptor harness
 /* globals user_pref */
 user_pref("dom.performance.time_to_non_blank_paint.enabled", true);
+
+// required for geckoview logging
+user_pref("geckoview.console.enabled", true);
--- a/testing/raptor/raptor/cmdline.py
+++ b/testing/raptor/raptor/cmdline.py
@@ -12,17 +12,17 @@ from mozlog.commandline import add_loggi
 def create_parser(mach_interface=False):
     parser = argparse.ArgumentParser()
     add_arg = parser.add_argument
 
     add_arg('-t', '--test', required=True, dest='test',
             help="name of raptor test to run")
     add_arg('--app', default='firefox', dest='app',
             help="name of the application we are testing (default: firefox)",
-            choices=['firefox', 'chrome'])
+            choices=['firefox', 'chrome', 'geckoview'])
     add_arg('-b', '--binary', dest='binary',
             help="path to the browser executable that we are testing")
     if not mach_interface:
         add_arg('--branchName', dest="branch_name", default='',
                 help="Name of the branch we are testing on")
         add_arg('--symbolsPath', dest='symbols_path',
                 help="Path to the symbols for the build we are testing")
         add_arg('--run-local', dest="run_local", default=False, action="store_true",
@@ -34,17 +34,19 @@ def create_parser(mach_interface=False):
     return parser
 
 
 def verify_options(parser, args):
     ctx = vars(args)
     if args.binary is None:
         parser.error("--binary is required!")
 
-    if not os.path.isfile(args.binary):
-        parser.error("{binary} does not exist!".format(**ctx))
+    # if running on a desktop browser make sure the binary exists
+    if args.app != "geckoview":
+        if not os.path.isfile(args.binary):
+            parser.error("{binary} does not exist!".format(**ctx))
 
 
 def parse_args(argv=None):
     parser = create_parser()
     args = parser.parse_args(argv)
     verify_options(parser, args)
     return args
--- a/testing/raptor/raptor/control_server.py
+++ b/testing/raptor/raptor/control_server.py
@@ -83,16 +83,18 @@ class RaptorControlServer():
     def __init__(self, results_handler):
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.server = None
         self._server_thread = None
         self.port = None
         self.results_handler = results_handler
         self.browser_proc = None
         self._finished = False
+        self.device = None
+        self.app_name = None
 
     def start(self):
         config_dir = os.path.join(here, 'tests')
         os.chdir(config_dir)
 
         # pick a free port
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.bind(('', 0))
@@ -107,26 +109,32 @@ class RaptorControlServer():
 
         self._server_thread = threading.Thread(target=httpd.serve_forever)
         self._server_thread.setDaemon(True)  # don't hang on exit
         self._server_thread.start()
         LOG.info("raptor control server running on port %d..." % self.port)
         self.server = httpd
 
     def shutdown_browser(self):
-        LOG.info("shutting down browser (pid: %d)" % self.browser_proc.pid)
+        if self.device is not None:
+            LOG.info("shutting down android app %s" % self.app_name)
+        else:
+            LOG.info("shutting down browser (pid: %d)" % self.browser_proc.pid)
         self.kill_thread = threading.Thread(target=self.wait_for_quit)
         self.kill_thread.daemon = True
         self.kill_thread.start()
 
     def wait_for_quit(self, timeout=15):
         """Wait timeout seconds for the process to exit. If it hasn't
         exited by then, kill it.
         """
-        self.browser_proc.wait(timeout)
-        if self.browser_proc.poll() is None:
-            self.browser_proc.kill()
+        if self.device is not None:
+            self.device.stop_application(self.app_name)
+        else:
+            self.browser_proc.wait(timeout)
+            if self.browser_proc.poll() is None:
+                self.browser_proc.kill()
         self._finished = True
 
     def stop(self):
         LOG.info("shutting down control server")
         self.server.shutdown()
         self._server_thread.join()
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -7,16 +7,17 @@ from __future__ import absolute_import
 
 import json
 import os
 import sys
 import time
 
 import mozinfo
 
+from mozdevice import ADBAndroid
 from mozlog import commandline, get_default_logger
 from mozprofile import create_profile
 from mozrunner import runners
 
 # need this so raptor imports work both from /raptor and via mach
 here = os.path.abspath(os.path.dirname(__file__))
 webext_dir = os.path.join(os.path.dirname(here), 'webext')
 sys.path.insert(0, here)
@@ -49,54 +50,74 @@ class Raptor(object):
         self.config['run_local'] = run_local
         self.config['obj_path'] = obj_path
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.log = get_default_logger(component='raptor-main')
         self.control_server = None
         self.playback = None
         self.benchmark = None
 
-        # Create the profile
-        self.profile = create_profile(self.config['app'])
+        # Create the profile; for geckoview we want a firefox profile type
+        if self.config['app'] == 'geckoview':
+            self.profile = create_profile('firefox')
+        else:
+            self.profile = create_profile(self.config['app'])
 
         # Merge in base profiles
         with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
             base_profiles = json.load(fh)['raptor']
 
         for name in base_profiles:
             path = os.path.join(self.profile_data_dir, name)
             self.log.info("Merging profile: {}".format(path))
             self.profile.merge(path)
 
         # create results holder
         self.results_handler = RaptorResultsHandler()
 
-        # Create the runner
-        self.output_handler = OutputHandler()
-        process_args = {
-            'processOutputLine': [self.output_handler],
-        }
-        runner_cls = runners[app]
-        self.runner = runner_cls(
-            binary, profile=self.profile, process_args=process_args)
+        # when testing desktop browsers we use mozrunner to start the browser; when
+        # testing on android (i.e. geckoview) we use mozdevice to control the device app
+
+        if self.config['app'] == "geckoview":
+            # create the android device handler; it gets initiated and sets up adb etc
+            self.log.info("creating android device handler using mozdevice")
+            self.device = ADBAndroid(verbose=True)
+            self.device.clear_logcat()
+        else:
+            # create the desktop browser runner
+            self.log.info("creating browser runner using mozrunner")
+            self.output_handler = OutputHandler()
+            process_args = {
+                'processOutputLine': [self.output_handler],
+            }
+            runner_cls = runners[app]
+            self.runner = runner_cls(
+                binary, profile=self.profile, process_args=process_args)
 
         self.log.info("raptor config: %s" % str(self.config))
 
     @property
     def profile_data_dir(self):
         if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
             return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
         if build:
             return os.path.join(build.topsrcdir, 'testing', 'profiles')
         return os.path.join(here, 'profile_data')
 
     def start_control_server(self):
         self.control_server = RaptorControlServer(self.results_handler)
         self.control_server.start()
 
+        # for android we must make the control server available to the device
+        if self.config['app'] == "geckoview":
+            self.log.info("making the raptor control server port available to device")
+            _tcp_port = "tcp:%s" % self.control_server.port
+            _cmd = ["reverse", _tcp_port, _tcp_port]
+            self.device.command_output(_cmd)
+
     def get_playback_config(self, test):
         self.config['playback_tool'] = test.get('playback')
         self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
         self.config['playback_binary_manifest'] = test.get('playback_binary_manifest', None)
         _key = 'playback_binary_zip_%s' % self.config['platform']
         self.config['playback_binary_zip'] = test.get(_key, None)
         self.config['playback_pageset_manifest'] = test.get('playback_pageset_manifest', None)
         _key = 'playback_pageset_zip_%s' % self.config['platform']
@@ -116,74 +137,121 @@ class Raptor(object):
         else:
             benchmark_port = 0
 
         gen_test_config(self.config['app'],
                         test['name'],
                         self.control_server.port,
                         benchmark_port)
 
+        # for android we must make the benchmarks server available to the device
+        if self.config['app'] == "geckoview":
+            self.log.info("making the raptor benchmarks server port available to device")
+            _tcp_port = "tcp:%s" % benchmark_port
+            _cmd = ["reverse", _tcp_port, _tcp_port]
+            self.device.command_output(_cmd)
+
         # must intall raptor addon each time because we dynamically update some content
         raptor_webext = os.path.join(webext_dir, 'raptor')
         self.log.info("installing webext %s" % raptor_webext)
         self.profile.addons.install(raptor_webext)
 
         # add test specific preferences
         if test.get("preferences", None) is not None:
             if self.config['app'] == "firefox":
                 self.profile.set_preferences(json.loads(test['preferences']))
             else:
                 self.log.info("preferences were configured for the test, \
                               but we do not install them on non Firefox browsers.")
 
         # on firefox we can get an addon id; chrome addon actually is just cmd line arg
-        if self.config['app'] == "firefox":
+        if self.config['app'] in ["firefox", "geckoview"]:
             webext_id = self.profile.addons.addon_details(raptor_webext)['id']
 
         # some tests require tools to playback the test pages
         if test.get('playback', None) is not None:
             self.get_playback_config(test)
             # startup the playback tool
             self.playback = get_playback(self.config)
 
-        self.runner.start()
+        # for geckoview we must copy the profile onto the device and set perms
+        if self.config['app'] == "geckoview":
+            self.log.info("copying firefox profile onto the android device")
+            self.device_profile = "/sdcard/raptor-profile"
+            if self.device.is_dir(self.device_profile):
+                self.device.rm(self.device_profile, recursive=True)
+
+            self.device.mkdir(self.device_profile)
+            self.device.push(self.profile.profile, self.device_profile)
+
+            self.log.info("setting permisions to profile dir on the device")
+            self.device.chmod(self.device_profile, recursive=True)
+
+            # now start the geckoview app
+            self.log.info("starting %s" % self.config['app'])
+
+            extra_args = ["-profile", self.device_profile,
+                          "--es", "env0", "LOG_VERBOSE=1",
+                          "--es", "env1", "R_LOG_LEVEL=6"]
 
-        proc = self.runner.process_handler
-        self.output_handler.proc = proc
-        self.control_server.browser_proc = proc
+            self.device.launch_activity(self.config['binary'],
+                                        "GeckoViewActivity",
+                                        extra_args=extra_args,
+                                        url='about:blank',
+                                        fail_if_running=False)
+
+            self.control_server.device = self.device
+            self.control_server.app_name = self.config['binary']
+
+        else:
+            # now start the desktop browser
+            self.log.info("starting %s" % self.config['app'])
+
+            self.runner.start()
+            proc = self.runner.process_handler
+            self.output_handler.proc = proc
+
+            self.control_server.browser_proc = proc
+
+        # set our cs flag to indicate we are running the browser/app
         self.control_server._finished = False
 
         # convert to seconds and account for page cycles
         timeout = int(timeout / 1000) * int(test['page_cycles'])
         try:
             elapsed_time = 0
             while not self.control_server._finished:
                 time.sleep(1)
                 elapsed_time += 1
                 if elapsed_time > (timeout) - 5:  # stop 5 seconds early
+                    self.log.info("application timed out after {} seconds".format(timeout))
                     self.control_server.wait_for_quit()
                     break
         finally:
-            try:
-                self.runner.check_for_crashes()
-            except NotImplementedError:  # not implemented for Chrome
-                pass
+            if self.config['app'] != "geckoview":
+                try:
+                    self.runner.check_for_crashes()
+                except NotImplementedError:  # not implemented for Chrome
+                    pass
+            # TODO: if on geckoview is there some cleanup here i.e. check for crashes?
 
         if self.playback is not None:
             self.playback.stop()
 
         # remove the raptor webext; as it must be reloaded with each subtest anyway
         # applies to firefox only; chrome the addon is actually just cmd line arg
-        if self.config['app'] == "firefox":
+        if self.config['app'] in ["firefox", "geckoview"]:
             self.log.info("removing webext %s" % raptor_webext)
             self.profile.addons.remove_addon(webext_id)
 
-        if self.runner.is_running():
-            self.log.info("Application timed out after {} seconds".format(timeout))
-            self.runner.stop()
+        if self.config['app'] != "geckoview":
+            if self.runner.is_running():
+                self.runner.stop()
+        # TODO the geckoview app should have been shutdown by this point by the
+        # control server, but we can double-check here to make sure
 
     def process_results(self):
         # when running locally output results in build/raptor.json; when running
         # in production output to a local.json to be turned into tc job artifact
         if self.config.get('run_local', False):
             if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
                 raptor_json_path = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
                                                 'testing', 'mozharness', 'build', 'raptor.json')
@@ -192,17 +260,18 @@ class Raptor(object):
         else:
             raptor_json_path = os.path.join(os.getcwd(), 'local.json')
 
         self.config['raptor_json_path'] = raptor_json_path
         return self.results_handler.summarize_and_output(self.config)
 
     def clean_up(self):
         self.control_server.stop()
-        self.runner.stop()
+        if self.config['app'] != "geckoview":
+            self.runner.stop()
         self.log.info("finished")
 
 
 def main(args=sys.argv[1:]):
     args = parse_args()
     commandline.setup_logging('raptor', args, {'tbpl': sys.stdout})
     LOG = get_default_logger(component='raptor-main')
 
--- a/testing/raptor/raptor/tests/raptor-speedometer.ini
+++ b/testing/raptor/raptor/tests/raptor-speedometer.ini
@@ -13,8 +13,13 @@ unit = score
 lower_is_better = false
 alert_threshold = 2.0
 
 [raptor-speedometer-firefox]
 apps = firefox
 
 [raptor-speedometer-chrome]
 apps = chrome
+
+[raptor-speedometer-geckoview]
+page_timeout = 1800000 # temp 30 min for debugging
+page_cycles = 1
+apps = geckoview
--- a/testing/raptor/requirements.txt
+++ b/testing/raptor/requirements.txt
@@ -1,4 +1,5 @@
 mozrunner ~= 7.0
 mozprofile ~= 1.1
 manifestparser >= 1.1
 wptserve ~= 1.4.0
+mozdevice ~= 1.0
--- a/testing/raptor/test/test_cmdline.py
+++ b/testing/raptor/test/test_cmdline.py
@@ -5,17 +5,17 @@ import pytest
 
 import mozunit
 
 from argparse import ArgumentParser, Namespace
 from raptor.cmdline import verify_options
 
 
 def test_verify_options(filedir):
-    args = Namespace(binary='invalid/path')
+    args = Namespace(app='firefox', binary='invalid/path')
     parser = ArgumentParser()
 
     with pytest.raises(SystemExit):
         verify_options(parser, args)
 
     args.binary = os.path.join(filedir, 'fake_binary.exe')
     verify_options(parser, args)  # assert no exception
 
--- a/testing/raptor/webext/raptor/benchmark-relay.js
+++ b/testing/raptor/webext/raptor/benchmark-relay.js
@@ -17,9 +17,10 @@ function receiveMessage(event) {
 function sendResult(_type, _value) {
   // send result back to background runner script
   console.log("sending result back to runner: " + _type + " " + _value);
   chrome.runtime.sendMessage({"type": _type, "value": _value}, function(response) {
     console.log(response.text);
   });
 }
 
+console.log("raptor benchmark-relay content loaded");
 window.addEventListener("message", receiveMessage);
--- a/testing/raptor/webext/raptor/runner.js
+++ b/testing/raptor/webext/raptor/runner.js
@@ -1,8 +1,9 @@
+
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 // this extension requires a 'control server' to be running on port 8000
 // (see raptor prototype framework). It will provide the test options, as
 // well as receive test results
 
@@ -100,17 +101,17 @@ function getTestSettings() {
             }
           } else {
             console.log("abort: 'measure' key not found in test settings");
             cleanUp();
           }
         }
 
         // write options to storage that our content script needs to know
-        if (browserName === "firefox") {
+        if (["firefox", "geckoview"].includes(browserName)) {
           ext.storage.local.clear().then(function() {
             ext.storage.local.set({settings}).then(function() {
               console.log("wrote settings to ext local storage");
               resolve();
             });
           });
         } else {
           ext.storage.local.clear(function() {
@@ -122,17 +123,17 @@ function getTestSettings() {
         }
       });
     });
   });
 }
 
 function getBrowserInfo() {
   return new Promise(resolve => {
-    if (browserName === "firefox") {
+    if (["firefox", "geckoview"].includes(browserName)) {
       ext = browser;
       var gettingInfo = browser.runtime.getBrowserInfo();
       gettingInfo.then(function(bi) {
         results.browser = bi.name + " " + bi.version + " " + bi.buildID;
         console.log("testing on " + results.browser);
         resolve();
       });
     } else {
@@ -152,17 +153,17 @@ function getBrowserInfo() {
 
 function testTabCreated(tab) {
   testTabID = tab.id;
   console.log("opened new empty tab " + testTabID);
   nextCycle();
 }
 
 async function testTabUpdated(tab) {
-  console.log("tab " + tab.id + " reloaded");
+  console.log("test tab updated");
   // wait for pageload test result from content
   await waitForResult();
   // move on to next cycle (or test complete)
   nextCycle();
 }
 
 function waitForResult() {
   console.log("awaiting results...");
@@ -210,17 +211,17 @@ function nextCycle() {
         }
         if (getFNBPaint)
           isFNBPaintPending = true;
         if (getFCP)
           isFCPPending = true;
       } else if (testType == "benchmark") {
         isBenchmarkPending = true;
       }
-      // (re)load the test page
+      // update the test page - browse to our test URL
       ext.tabs.update(testTabID, {url: testURL}, testTabUpdated);
     }, pageCycleDelay);
   } else {
     verifyResults();
   }
 }
 
 function timeoutAlarmListener(alarm) {
@@ -236,17 +237,17 @@ function setTimeoutAlarm(timeoutName, ti
   var now = Date.now(); // eslint-disable-line mozilla/avoid-Date-timing
   var timeout_when = now + timeoutMS;
   ext.alarms.create(timeoutName, { when: timeout_when });
   console.log("now is " + now + ", set raptor alarm " +
               timeoutName + " to expire at " + timeout_when);
 }
 
 function cancelTimeoutAlarm(timeoutName) {
-  if (browserName === "firefox") {
+  if (browserName === "firefox" || browserName === "geckoview") {
     var clearAlarm = ext.alarms.clear(timeoutName);
     clearAlarm.then(function(onCleared) {
       if (onCleared) {
         console.log("cancelled " + timeoutName);
       } else {
         console.error("failed to clear " + timeoutName);
       }
     });
@@ -354,16 +355,17 @@ function cleanUp() {
     console.log("benchmark complete");
   }
   window.onload = null;
   // tell the control server we are done and the browser can be shutdown
   postToControlServer("status", "__raptor_shutdownBrowser");
 }
 
 function runner() {
+  console.log("Welcome to Jurassic Park!");
   let config = getTestConfig();
   console.log("test name is: " + config.test_name);
   console.log("test settings url is: " + config.test_settings_url);
   testName = config.test_name;
   settingsURL = config.test_settings_url;
   csPort = config.cs_port;
   browserName = config.browser;
   benchmarkPort = config.benchmark_port;
@@ -383,14 +385,20 @@ function runner() {
       ext.tabs.onCreated.addListener(testTabCreated);
       // timeout alarm listener
       ext.alarms.onAlarm.addListener(timeoutAlarmListener);
 
       // create new empty tab, which starts the test; we want to
       // wait some time for the browser to settle before beginning
       var text = "* pausing " + postStartupDelay / 1000 + " seconds to let browser settle... *";
       postToControlServer("status", text);
-      setTimeout(function() { ext.tabs.create({url: "about:blank"}); }, postStartupDelay);
+
+      // on geckoview you can't create a new tab; only using existing tab - set it blank first
+      if (config.browser == "geckoview") {
+        setTimeout(function() { nextCycle(); }, postStartupDelay);
+      } else {
+        setTimeout(function() { ext.tabs.create({url: "about:blank"}); }, postStartupDelay);
+      }
     });
   });
 }
 
 window.onload = runner();
--- a/third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js
+++ b/third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js
@@ -83,16 +83,17 @@
                     values.push(vals);
                     for (var count=0; count < vals.length; count ++) {
                         allNames.push(fullNames[i]);
                     }
                 }
 
                 if (location.search == '?raptor') {
                     _data = ['raptor-benchmark', 'speedometer', measuredValuesByFullName];
+                    console.log('speedometer source about to post results to the raptor webext');
                     window.postMessage(_data, '*');
                 } else {
                     tpRecordTime(values.join(','), 0, allNames.join(','));
                 }
             } else {
                 for (var i = 0; i < fullNames.length; i++) {
                     var values = measuredValuesByFullName[fullNames[i]];
                     PerfTestRunner.reportValues(createTest(fullNames[i], values.aggregator, i + 1 == fullNames.length), values);