Bug 1384245 - Run gtest benchmarks multiple times and take median r=milan
authorWilliam Lachance <wlachance@mozilla.com>
Mon, 31 Jul 2017 12:47:32 -0400
changeset 423149 7ac289e62574bc5c0758874c7f14d84edd5bd178
parent 423148 2da66805c3d13861f55c84ddcb48b06da090a443
child 423150 8c881a1185673e8a6bcb9481177663604e442d39
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmilan
bugs1384245
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1384245 - Run gtest benchmarks multiple times and take median r=milan MozReview-Commit-ID: 3g3RB36Brpw
testing/gtest/mozilla/MozGTestBench.cpp
--- a/testing/gtest/mozilla/MozGTestBench.cpp
+++ b/testing/gtest/mozilla/MozGTestBench.cpp
@@ -1,41 +1,60 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "MozGTestBench.h"
 #include "mozilla/TimeStamp.h"
 #include <stdio.h>
+#include <string>
+#include <vector>
 
 #define MOZ_GTEST_BENCH_FRAMEWORK "platform_microbench"
+#define MOZ_GTEST_NUM_ITERATIONS 5
 
 using mozilla::TimeStamp;
 
 namespace mozilla {
 void GTestBench(const char* aSuite, const char* aName,
                 const std::function<void()>& aTest)
 {
 #ifdef DEBUG
   // Run the test to make sure that it doesn't fail but don't log
   // any measurements since it's not an optimized build.
   aTest();
 #else
   bool shouldAlert = bool(getenv("PERFHERDER_ALERTING_ENABLED"));
+  std::vector<int> durations;
 
-  mozilla::TimeStamp start = TimeStamp::Now();
+  for (int i=0; i<MOZ_GTEST_NUM_ITERATIONS; i++) {
+    mozilla::TimeStamp start = TimeStamp::Now();
+
+    aTest();
+
+    durations.push_back((TimeStamp::Now() - start).ToMicroseconds());
+  }
 
-  aTest();
+  std::string replicatesStr = "[" + std::to_string(durations[0]);
+  for (int i=1; i<MOZ_GTEST_NUM_ITERATIONS; i++) {
+    replicatesStr += "," + std::to_string(durations[i]);
+  }
+  replicatesStr += "]";
 
-  int msDuration = (TimeStamp::Now() - start).ToMicroseconds();
+  // median is at index floor(i/2) if number of replicates is odd,
+  // (i/2-1) if even
+  std::sort(durations.begin(), durations.end());
+  int medianIndex = (MOZ_GTEST_NUM_ITERATIONS / 2) + ((durations.size() % 2 == 0) ? (-1) : 0);
 
   // Print the result for each test. Let perfherder aggregate for us
   printf("PERFHERDER_DATA: {\"framework\": {\"name\": \"%s\"}, "
-                       "\"suites\": [{\"name\": \"%s\", \"subtests\": "
-                           "[{\"name\": \"%s\", \"value\": %i, \"lowerIsBetter\": true, \"shouldAlert\": %s}]"
-                       "}]}\n",
-                       MOZ_GTEST_BENCH_FRAMEWORK, aSuite, aName, msDuration, shouldAlert ? "true" : "false");
+         "\"suites\": [{\"name\": \"%s\", \"subtests\": "
+         "[{\"name\": \"%s\", \"value\": %i, \"replicates\": %s, "
+         "\"lowerIsBetter\": true, \"shouldAlert\": %s}]"
+         "}]}\n",
+         MOZ_GTEST_BENCH_FRAMEWORK, aSuite, aName, durations[medianIndex],
+         replicatesStr.c_str(), shouldAlert ? "true" : "false");
 #endif
 }
 
 } // mozilla