Bug 1461652 - Mock gzip compression in tests to simulate large payloads. r=chutten
☠☠ backed out by 651a5ed46d2d ☠ ☠
authorJan-Erik Rediger <jrediger@mozilla.com>
Wed, 16 May 2018 02:21:00 -0400
changeset 418603 5edae29aca3248c223b84447ebfba78f355e3f8b
parent 418602 f050b09ee268eda98757cceb75b9e06a357a68d4
child 418604 8d8c17bc31455084017d5fa5a3ecb1c98f448cfa
push id34007
push usercsabou@mozilla.com
push dateThu, 17 May 2018 09:47:02 +0000
treeherdermozilla-central@8fb36531f7d0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschutten
bugs1461652
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1461652 - Mock gzip compression in tests to simulate large payloads. r=chutten MozReview-Commit-ID: Gct9oVfPVou
toolkit/components/telemetry/TelemetrySend.jsm
toolkit/components/telemetry/tests/unit/head.js
toolkit/components/telemetry/tests/unit/test_TelemetrySend.js
--- a/toolkit/components/telemetry/TelemetrySend.jsm
+++ b/toolkit/components/telemetry/TelemetrySend.jsm
@@ -104,16 +104,17 @@ const XHR_ERROR_TYPE = [
  * that would otherwise be very hard to cover.
  */
 var Policy = {
   now: () => new Date(),
   midnightPingFuzzingDelay: () => MIDNIGHT_FUZZING_DELAY_MS,
   pingSubmissionTimeout: () => PING_SUBMIT_TIMEOUT_MS,
   setSchedulerTickTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
   clearSchedulerTickTimeout: (id) => clearTimeout(id),
+  gzipCompressString: gzipCompressString,
 };
 
 /**
  * Determine if the ping has the new v4 ping format or the legacy v2 one or earlier.
  */
 function isV4PingFormat(aPing) {
   return ("id" in aPing) && ("application" in aPing) &&
          ("version" in aPing) && (aPing.version >= 2);
@@ -1209,17 +1210,17 @@ var TelemetrySendImpl = {
     let startTime = Utils.monotonicNow();
     let utf8Payload = converter.ConvertFromUnicode(JSON.stringify(networkPayload));
     utf8Payload += converter.Finish();
     Telemetry.getHistogramById("TELEMETRY_STRINGIFY").add(Utils.monotonicNow() - startTime);
 
     let payloadStream = Cc["@mozilla.org/io/string-input-stream;1"]
                         .createInstance(Ci.nsIStringInputStream);
     startTime = Utils.monotonicNow();
-    payloadStream.data = gzipCompressString(utf8Payload);
+    payloadStream.data = Policy.gzipCompressString(utf8Payload);
 
     // Check the size and drop pings which are too big.
     const compressedPingSizeBytes = payloadStream.data.length;
     if (compressedPingSizeBytes > TelemetryStorage.MAXIMUM_PING_SIZE) {
       this._log.error("_doPing - submitted ping exceeds the size limit, size: " + compressedPingSizeBytes);
       Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND").add();
       Telemetry.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB")
                .add(Math.floor(compressedPingSizeBytes / 1024 / 1024));
--- a/toolkit/components/telemetry/tests/unit/head.js
+++ b/toolkit/components/telemetry/tests/unit/head.js
@@ -270,16 +270,27 @@ function fakeGeneratePingId(func) {
   module.Policy.generatePingId = func;
 }
 
 function fakeCachedClientId(uuid) {
   let module = ChromeUtils.import("resource://gre/modules/TelemetryController.jsm", {});
   module.Policy.getCachedClientID = () => uuid;
 }
 
+// Fake the gzip compression for the next ping to be sent out
+// and immediately reset to the original function.
+function fakeGzipCompressStringForNextPing(length) {
+  let send = ChromeUtils.import("resource://gre/modules/TelemetrySend.jsm", {});
+  let largePayload = generateString(length);
+  send.Policy.gzipCompressString = (data) => {
+    send.Policy.gzipCompressString = send.gzipCompressString;
+    return largePayload;
+  };
+}
+
 // Return a date that is |offset| ms in the future from |date|.
 function futureDate(date, offset) {
   return new Date(date.getTime() + offset);
 }
 
 function truncateToDays(aMsec) {
   return Math.floor(aMsec / MILLISECONDS_PER_DAY);
 }
@@ -296,16 +307,20 @@ function generateRandomString(length) {
 
   while (string.length < length) {
     string += Math.random().toString(36);
   }
 
   return string.substring(0, length);
 }
 
+function generateString(length) {
+  return new Array(length + 1).join("a");
+}
+
 // Short-hand for retrieving the histogram with that id.
 function getHistogram(histogramId) {
   return Telemetry.getHistogramById(histogramId);
 }
 
 // Short-hand for retrieving the snapshot of the Histogram with that id.
 function getSnapshot(histogramId) {
   return Telemetry.getHistogramById(histogramId).snapshot();
--- a/toolkit/components/telemetry/tests/unit/test_TelemetrySend.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetrySend.js
@@ -329,33 +329,33 @@ add_task(async function test_discardBigP
   let histDiscardedSize = Telemetry.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB");
   let histSuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS");
   let histSendTimeSuccess = Telemetry.getHistogramById("TELEMETRY_SEND_SUCCESS");
   let histSendTimeFail = Telemetry.getHistogramById("TELEMETRY_SEND_FAILURE");
   for (let h of [histSizeExceeded, histDiscardedSize, histSuccess, histSendTimeSuccess, histSendTimeFail]) {
     h.clear();
   }
 
-  // Generate a 2MB string and create an oversized payload.
-  const OVERSIZED_PAYLOAD = {"data": generateRandomString(4 * 1024 * 1024)};
-
   // Submit a ping of a normal size and check that we don't count it in the histogram.
   await TelemetryController.submitExternalPing(TEST_PING_TYPE, { test: "test" });
   await TelemetrySend.testWaitOnOutgoingPings();
   await PingServer.promiseNextPing();
 
   Assert.equal(histSizeExceeded.snapshot().sum, 0, "Telemetry must report no oversized ping submitted.");
   Assert.equal(histDiscardedSize.snapshot().sum, 0, "Telemetry must report no oversized pings.");
   Assert.deepEqual(histSuccess.snapshot().counts, [0, 1, 0], "Should have recorded sending success.");
   Assert.equal(histogramValueCount(histSendTimeSuccess.snapshot()), 1, "Should have recorded send success time.");
   Assert.greaterOrEqual(histSendTimeSuccess.snapshot().sum, 0, "Should have recorded send success time.");
   Assert.equal(histogramValueCount(histSendTimeFail.snapshot()), 0, "Should not have recorded send failure time.");
 
   // Submit an oversized ping and check that it gets discarded.
   TelemetryHealthPing.testReset();
+  // Ensure next ping has a 2 MB gzipped payload.
+  fakeGzipCompressStringForNextPing(2 * 1024 * 1024);
+  const OVERSIZED_PAYLOAD = {"data": "empty on purpose - policy takes care of size"};
   await TelemetryController.submitExternalPing(TEST_PING_TYPE, OVERSIZED_PAYLOAD);
   await TelemetrySend.testWaitOnOutgoingPings();
   let ping = await PingServer.promiseNextPing();
 
   Assert.equal(ping.type, TelemetryHealthPing.HEALTH_PING_TYPE, "Should have received a health ping.");
   Assert.equal(ping.payload.reason, TelemetryHealthPing.Reason.IMMEDIATE, "Health ping should have the right reason.");
   Assert.deepEqual(ping.payload[TelemetryHealthPing.FailureType.DISCARDED_FOR_SIZE],
     {[TEST_PING_TYPE]: 1}, "Should have recorded correct type of oversized ping.");
@@ -371,22 +371,23 @@ add_task(async function test_discardBigP
 });
 
 add_task(async function test_largeButWithinLimit() {
   const TEST_PING_TYPE = "test-ping-type";
 
   let histSuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS");
   histSuccess.clear();
 
-  // Generate a 900KB string and a large payload that is still within the 1MB limit.
-  const LARGE_PAYLOAD = {"data": generateRandomString(900 * 1024)};
+  // Next ping will have a 900KB gzip payload.
+  fakeGzipCompressStringForNextPing(900 * 1024);
+  const LARGE_PAYLOAD = {"data": "empty on purpose - policy takes care of size"};
 
   await TelemetryController.submitExternalPing(TEST_PING_TYPE, LARGE_PAYLOAD);
   await TelemetrySend.testWaitOnOutgoingPings();
-  await PingServer.promiseNextPing();
+  await PingServer.promiseNextRequest();
 
   Assert.deepEqual(histSuccess.snapshot().counts, [0, 1, 0], "Should have sent large ping.");
 });
 
 add_task(async function test_evictedOnServerErrors() {
   const TEST_TYPE = "test-evicted";
 
   await TelemetrySend.reset();