Bug 1050851 - Add long running steeplechase tests from the sunny day QA lab environment. r=gmealer
authorNils Ohlmeier [:drno] <drno@ohlmeier.org>
Fri, 08 Aug 2014 10:57:00 -0400
changeset 198756 affd7637e02e7e918078afa3e29b0ac85b4bf5cf
parent 198755 0e04c828071d75c7587fee91aa9eaf738cc31eee
child 198757 7079b7552946f5e7d274060206a785c340abbf1d
push id27284
push userryanvm@gmail.com
push dateSat, 09 Aug 2014 15:25:31 +0000
treeherdermozilla-central@ad8cb646fad6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgmealer
bugs1050851
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1050851 - Add long running steeplechase tests from the sunny day QA lab environment. r=gmealer
dom/media/tests/mochitest/long.js
dom/media/tests/mochitest/steeplechase_long.ini
dom/media/tests/mochitest/test_peerConnection_basicAudioVideoCombined_long.html
dom/media/tests/mochitest/test_peerConnection_basicAudio_long.html
dom/media/tests/mochitest/test_peerConnection_basicVideo_long.html
--- a/dom/media/tests/mochitest/long.js
+++ b/dom/media/tests/mochitest/long.js
@@ -54,16 +54,20 @@ function outputPcStats(stats, label) {
     }
   }
   outputStr += '\n\n';
   dump(outputStr);
 }
 
 
 var _lastStats = {};
+
+const MAX_ERROR_CYCLES = 5;
+var _errorCount = {};
+
 /**
  * Verifies the peer connection stats interval over interval
  *
  * @param {Object} stats
  *        Stats object to use for verification
  * @param {string} label
  *        Identifies the peer connection. Differentiates stats for
  *        interval-over-interval verification in cases where more than one set
@@ -76,38 +80,58 @@ function verifyPcStats(stats, label) {
   ];
 
   const INCREASING_OUTBOUND_STAT_NAMES = [
     'bytesSent',
     'packetsSent'
   ];
 
   if (_lastStats[label] !== undefined) {
+    var errorsInCycle = false;
+
     function verifyIncrease(rtpName, statNames) {
       var timestamp = new Date(stats[rtpName].timestamp).toISOString();
 
       statNames.forEach(function (statName) {
-        ok(stats[rtpName][statName] > _lastStats[label][rtpName][statName],
+        var passed = stats[rtpName][statName] >
+            _lastStats[label][rtpName][statName];
+        if (!passed) {
+          errorsInCycle = true;
+        }
+        ok(passed,
            timestamp + '.' + label + '.' + rtpName + '.' + statName,
            label + '.' + rtpName + '.' + statName + ' increased (value=' +
            stats[rtpName][statName] + ')');
       });
     }
 
     for (var prop in stats) {
       if (isLocalRtp(stats[prop])) {
         if (stats[prop].type === 'inboundrtp') {
           verifyIncrease(prop, INCREASING_INBOUND_STAT_NAMES);
         } else {
           verifyIncrease(prop, INCREASING_OUTBOUND_STAT_NAMES);
         }
       }
     }
+
+    if (errorsInCycle) {
+      _errorCount[label] += 1;
+      info(label +": increased error counter to " + _errorCount[label]);
+    } else {
+      // looks like we recovered from a temp glitch
+      if (_errorCount[label] > 0) {
+        info(label + ": reseting error counter to zero");
+      }
+      _errorCount[label] = 0;
+    }
+  } else {
+    _errorCount[label] = 0;
   }
-  
+
   _lastStats[label] = stats;
 }
 
 
 /**
  * Retrieves and performs a series of operations on PeerConnection stats
  *
  * @param {PeerConnectionWrapper} pc
@@ -170,18 +194,26 @@ function generateIntervalCommand(callbac
     name,
     function (test) {
       var startTime = Date.now();
       var intervalId = setInterval(function () {
         if (callback) {
           callback(test);
         }
 
+        var failed = false;
+        Object.keys(_errorCount).forEach(function (label) {
+          if (_errorCount[label] > MAX_ERROR_CYCLES) {
+            ok(false, "Encountered more then " + MAX_ERROR_CYCLES + " cycles" +
+              " with errors on " + label);
+            failed = true;
+          }
+        });
         var timeElapsed = Date.now() - startTime;
-        if (timeElapsed >= duration) {
+        if ((timeElapsed >= duration) || failed) {
           clearInterval(intervalId);
           test.next();
         }
       }, interval);
     }
   ]
 }
 
--- a/dom/media/tests/mochitest/steeplechase_long.ini
+++ b/dom/media/tests/mochitest/steeplechase_long.ini
@@ -2,10 +2,12 @@
 support-files =
   head.js
   long.js
   mediaStreamPlayback.js
   pc.js
   templates.js
   turnConfig.js
 
+[test_peerConnection_basicAudio_long.html]
+[test_peerConnection_basicVideo_long.html]
 [test_peerConnection_basicAudioVideoCombined_long.html]
 
--- a/dom/media/tests/mochitest/test_peerConnection_basicAudioVideoCombined_long.html
+++ b/dom/media/tests/mochitest/test_peerConnection_basicAudioVideoCombined_long.html
@@ -15,29 +15,30 @@
   <script type="application/javascript" src="templates.js"></script>
   <script type="application/javascript" src="turnConfig.js"></script>
 </head>
 <body>
 <pre id="test">
 <script type="application/javascript">
   createHTML({
     bug: "1014328",
-    title: "Basic audio/video (combined) peer connection, long running"
+    title: "Basic audio/video (combined) peer connection, long running",
+    visible: true
   });
 
-  FAKE_ENABLED = false;
-
   var test;
   runTest(function (options) {
     options = options || {};
     options.commands = commandsPeerConnection.slice(0);
-    options.commands.push(generateIntervalCommand(verifyConnectionStatus));
+    options.commands.push(generateIntervalCommand(verifyConnectionStatus,
+                                                  1000 * 10,
+                                                  1000 * 3600 * 3));
 
     test = new PeerConnectionTest(options);
-    test.setMediaConstraints([{audio: true, video: true}],
-                             [{audio: true, video: true}]);
+    test.setMediaConstraints([{audio: true, video: true, fake: false}],
+                             [{audio: true, video: true, fake: false}]);
     test.run();
   });
 </script>
 </pre>
 </body>
 </html>
 
new file mode 100644
--- /dev/null
+++ b/dom/media/tests/mochitest/test_peerConnection_basicAudio_long.html
@@ -0,0 +1,43 @@
+<!DOCTYPE HTML>
+
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="application/javascript" src="head.js"></script>
+  <script type="application/javascript" src="long.js"></script>
+  <script type="application/javascript" src="mediaStreamPlayback.js"></script>
+  <script type="application/javascript" src="pc.js"></script>
+  <script type="application/javascript" src="templates.js"></script>
+  <script type="application/javascript" src="turnConfig.js"></script>
+</head>
+<body>
+<pre id="test">
+<script type="application/javascript">
+  createHTML({
+    bug: "796892",
+    title: "Basic audio-only peer connection",
+    visible: true
+  });
+
+  var test;
+  runNetworkTest(function (options) {
+    options = options || {};
+    options.commands = commandsPeerConnection.slice(0);
+    options.commands.push(generateIntervalCommand(verifyConnectionStatus,
+                                                  1000 * 10,
+                                                  1000 * 3600 * 3));
+
+    test = new PeerConnectionTest(options);
+    test.setMediaConstraints([{audio: true, fake: false}],
+                             [{audio: true, fake: false}]);
+    test.run();
+  });
+</script>
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/dom/media/tests/mochitest/test_peerConnection_basicVideo_long.html
@@ -0,0 +1,43 @@
+<!DOCTYPE HTML>
+
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<html>
+<head>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="application/javascript" src="head.js"></script>
+  <script type="application/javascript" src="long.js"></script>
+  <script type="application/javascript" src="mediaStreamPlayback.js"></script>
+  <script type="application/javascript" src="pc.js"></script>
+  <script type="application/javascript" src="templates.js"></script>
+  <script type="application/javascript" src="turnConfig.js"></script>
+</head>
+<body>
+<pre id="test">
+<script type="application/javascript">
+  createHTML({
+    bug: "796888",
+    title: "Basic video-only peer connection",
+    visible: true
+  });
+
+  var test;
+  runNetworkTest(function (options) {
+    options = options || {};
+    options.commands = commandsPeerConnection.slice(0);
+    options.commands.push(generateIntervalCommand(verifyConnectionStatus,
+                                                  1000 * 10,
+                                                  1000 * 3600 * 3));
+
+    test = new PeerConnectionTest(options);
+    test.setMediaConstraints([{video: true, fake: false}],
+                             [{video: true, fake: false}]);
+    test.run();
+  });
+</script>
+</pre>
+</body>
+</html>