Bug 1291199: Retry subprocess perf tests on failure. r=aswan
authorKris Maglione <maglione.k@gmail.com>
Tue, 02 Aug 2016 15:37:34 -0700
changeset 308311 79b31ab92faac9a6aae28104d3348bcf453ec146
parent 308310 2eb52e6d9b1894210f8d093b62f3ab840648bdb5
child 308312 4d27015914702b520c77574e82ba7f09e5b13f8d
push id31092
push usercbook@mozilla.com
push dateFri, 05 Aug 2016 10:16:59 +0000
treeherderautoland@b97dd7dd3cb9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaswan
bugs1291199
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1291199: Retry subprocess perf tests on failure. r=aswan MozReview-Commit-ID: 9PDT9tcYFqf
toolkit/components/extensions/test/xpcshell/test_ext_native_messaging_perf.js
toolkit/modules/subprocess/test/xpcshell/test_subprocess.js
--- a/toolkit/components/extensions/test/xpcshell/test_ext_native_messaging_perf.js
+++ b/toolkit/components/extensions/test/xpcshell/test_ext_native_messaging_perf.js
@@ -5,16 +5,17 @@
 XPCOMUtils.defineLazyModuleGetter(this, "MockRegistry",
                                   "resource://testing-common/MockRegistry.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "OS",
                                   "resource://gre/modules/osfile.jsm");
 
 Cu.import("resource://gre/modules/Subprocess.jsm");
 
 const MAX_ROUND_TRIP_TIME_MS = AppConstants.DEBUG || AppConstants.ASAN ? 36 : 18;
+const MAX_RETRIES = 5;
 
 
 const ECHO_BODY = String.raw`
   import struct
   import sys
 
   while True:
       rawlen = sys.stdin.read(4)
@@ -38,77 +39,89 @@ const SCRIPTS = [
 
 add_task(function* setup() {
   yield setupHosts(SCRIPTS);
 });
 
 add_task(function* test_round_trip_perf() {
   let extension = ExtensionTestUtils.loadExtension({
     background() {
-      let port = browser.runtime.connectNative("echo");
-
-      function next() {
-        port.postMessage({
-          "Lorem": {
-            "ipsum": {
-              "dolor": [
-                "sit amet",
-                "consectetur adipiscing elit",
-                "sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
-              ],
-              "Ut enim": [
-                "ad minim veniam",
-                "quis nostrud exercitation ullamco",
-                "laboris nisi ut aliquip ex ea commodo consequat.",
-              ],
-              "Duis": [
-                "aute irure dolor in reprehenderit in",
-                "voluptate velit esse cillum dolore eu",
-                "fugiat nulla pariatur.",
-              ],
-              "Excepteur": [
-                "sint occaecat cupidatat non proident",
-                "sunt in culpa qui officia deserunt",
-                "mollit anim id est laborum.",
-              ],
-            },
-          },
-        });
-      }
-
-      const COUNT = 1000;
-      let now;
-      function finish() {
-        let roundTripTime = (Date.now() - now) / COUNT;
-
-        browser.test.sendMessage("result", roundTripTime);
-      }
-
-      let count = 0;
-      port.onMessage.addListener(msg => {
-        if (count == 0) {
-          // Skip the first round, since it includes the time it takes
-          // the app to start up.
-          now = Date.now();
+      browser.test.onMessage.addListener(msg => {
+        if (msg != "run-tests") {
+          return;
         }
 
-        if (count++ <= COUNT) {
-          next();
-        } else {
-          finish();
+        let port = browser.runtime.connectNative("echo");
+
+        function next() {
+          port.postMessage({
+            "Lorem": {
+              "ipsum": {
+                "dolor": [
+                  "sit amet",
+                  "consectetur adipiscing elit",
+                  "sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
+                ],
+                "Ut enim": [
+                  "ad minim veniam",
+                  "quis nostrud exercitation ullamco",
+                  "laboris nisi ut aliquip ex ea commodo consequat.",
+                ],
+                "Duis": [
+                  "aute irure dolor in reprehenderit in",
+                  "voluptate velit esse cillum dolore eu",
+                  "fugiat nulla pariatur.",
+                ],
+                "Excepteur": [
+                  "sint occaecat cupidatat non proident",
+                  "sunt in culpa qui officia deserunt",
+                  "mollit anim id est laborum.",
+                ],
+              },
+            },
+          });
         }
-      });
+
+        const COUNT = 1000;
+        let now;
+        function finish() {
+          let roundTripTime = (Date.now() - now) / COUNT;
+
+          port.disconnect();
+          browser.test.sendMessage("result", roundTripTime);
+        }
 
-      next();
+        let count = 0;
+        port.onMessage.addListener(msg => {
+          if (count == 0) {
+            // Skip the first round, since it includes the time it takes
+            // the app to start up.
+            now = Date.now();
+          }
+
+          if (count++ <= COUNT) {
+            next();
+          } else {
+            finish();
+          }
+        });
+
+        next();
+      });
     },
     manifest: {
       permissions: ["nativeMessaging"],
     },
   }, ID);
 
   yield extension.startup();
 
-  let roundTripTime = yield extension.awaitMessage("result");
+  let roundTripTime = Infinity;
+  for (let i = 0; i < MAX_RETRIES && roundTripTime > MAX_ROUND_TRIP_TIME_MS; i++) {
+    extension.sendMessage("run-tests");
+    roundTripTime = yield extension.awaitMessage("result");
+  }
+
+  yield extension.unload();
+
   ok(roundTripTime <= MAX_ROUND_TRIP_TIME_MS,
      `Expected round trip time (${roundTripTime}ms) to be less than ${MAX_ROUND_TRIP_TIME_MS}ms`);
-
-  yield extension.unload();
 });
--- a/toolkit/modules/subprocess/test/xpcshell/test_subprocess.js
+++ b/toolkit/modules/subprocess/test/xpcshell/test_subprocess.js
@@ -3,16 +3,17 @@
 Cu.import("resource://gre/modules/AppConstants.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/Timer.jsm");
 
 
 const env = Cc["@mozilla.org/process/environment;1"].getService(Ci.nsIEnvironment);
 
 const MAX_ROUND_TRIP_TIME_MS = AppConstants.DEBUG || AppConstants.ASAN ? 18 : 9;
+const MAX_RETRIES = 5;
 
 let PYTHON;
 let PYTHON_BIN;
 let PYTHON_DIR;
 
 const TEST_SCRIPT = do_get_file("data_test_script.py").path;
 
 let read = pipe => {
@@ -177,48 +178,52 @@ add_task(function* test_subprocess_huge(
 
   let {exitCode} = yield proc.wait();
 
   equal(exitCode, 0, "Got expected exit code");
 });
 
 
 add_task(function* test_subprocess_round_trip_perf() {
-  let proc = yield Subprocess.call({
-    command: PYTHON,
-    arguments: ["-u", TEST_SCRIPT, "echo"],
-  });
+  let roundTripTime = Infinity;
+  for (let i = 0; i < MAX_RETRIES && roundTripTime > MAX_ROUND_TRIP_TIME_MS; i++) {
+    let proc = yield Subprocess.call({
+      command: PYTHON,
+      arguments: ["-u", TEST_SCRIPT, "echo"],
+    });
 
 
-  const LINE = "I'm a leaf on the wind.\n";
+    const LINE = "I'm a leaf on the wind.\n";
+
+    let now = Date.now();
+    const COUNT = 1000;
+    for (let i = 0; i < COUNT; i++) {
+      let [output] = yield Promise.all([
+        read(proc.stdout),
+        proc.stdin.write(LINE),
+      ]);
 
-  let now = Date.now();
-  const COUNT = 1000;
-  for (let i = 0; i < COUNT; i++) {
-    let [output] = yield Promise.all([
-      read(proc.stdout),
-      proc.stdin.write(LINE),
-    ]);
+      // We don't want to log this for every iteration, but we still need
+      // to fail if it goes wrong.
+      if (output !== LINE) {
+        equal(output, LINE, "Got expected output");
+      }
+    }
 
-    // We don't want to log this for every iteration, but we still need
-    // to fail if it goes wrong.
-    if (output !== LINE) {
-      equal(output, LINE, "Got expected output");
-    }
+    roundTripTime = (Date.now() - now) / COUNT;
+
+    yield proc.stdin.close();
+
+    let {exitCode} = yield proc.wait();
+
+    equal(exitCode, 0, "Got expected exit code");
   }
 
-  let roundTripTime = (Date.now() - now) / COUNT;
   ok(roundTripTime <= MAX_ROUND_TRIP_TIME_MS,
      `Expected round trip time (${roundTripTime}ms) to be less than ${MAX_ROUND_TRIP_TIME_MS}ms`);
-
-  yield proc.stdin.close();
-
-  let {exitCode} = yield proc.wait();
-
-  equal(exitCode, 0, "Got expected exit code");
 });
 
 
 add_task(function* test_subprocess_stderr_default() {
   const LINE1 = "I'm a leaf on the wind.\n";
   const LINE2 = "Watch how I soar.\n";
 
   let proc = yield Subprocess.call({