Bug 854018 - Record counts for FHR upload actions; r=rnewman
authorGregory Szorc <gps@mozilla.com>
Wed, 01 May 2013 09:41:55 -0700
changeset 130594 eb460fae0ece75f8f284be8f9618cc085466987f
parent 130465 4ff1e574e509f010eba959897491518334689db7
child 130595 3a7209e3c338c0f50fd919b80d70a9512b725ca4
push id27443
push userryanvm@gmail.com
push dateThu, 02 May 2013 11:39:45 +0000
treeherdermozilla-inbound@ff9eed6225d7 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrnewman
bugs854018
milestone23.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 854018 - Record counts for FHR upload actions; r=rnewman
services/healthreport/HealthReport.jsm
services/healthreport/HealthReportComponents.manifest
services/healthreport/healthreporter.jsm
services/healthreport/providers.jsm
services/healthreport/tests/xpcshell/test_healthreporter.js
--- a/services/healthreport/HealthReport.jsm
+++ b/services/healthreport/HealthReport.jsm
@@ -4,16 +4,17 @@
 
 "use strict";
 
 this.EXPORTED_SYMBOLS = [
   "HealthReporter",
   "AddonsProvider",
   "AppInfoProvider",
   "CrashesProvider",
+  "HealthReportProvider",
   "Metrics",
   "PlacesProvider",
   "ProfileMetadataProvider",
   "SearchesProvider",
   "SessionsProvider",
   "SysInfoProvider",
 ];
 
--- a/services/healthreport/HealthReportComponents.manifest
+++ b/services/healthreport/HealthReportComponents.manifest
@@ -1,12 +1,13 @@
 # Register Firefox Health Report providers.
 category healthreport-js-provider-default AddonsProvider resource://gre/modules/HealthReport.jsm
 category healthreport-js-provider-default AppInfoProvider resource://gre/modules/HealthReport.jsm
 category healthreport-js-provider-default CrashesProvider resource://gre/modules/HealthReport.jsm
+category healthreport-js-provider-default HealthReportProvider resource://gre/modules/HealthReport.jsm
 category healthreport-js-provider-default PlacesProvider resource://gre/modules/HealthReport.jsm
 category healthreport-js-provider-default ProfileMetadataProvider resource://gre/modules/HealthReport.jsm
 category healthreport-js-provider-default SearchesProvider resource://gre/modules/HealthReport.jsm
 category healthreport-js-provider-default SessionsProvider resource://gre/modules/HealthReport.jsm
 category healthreport-js-provider-default SysInfoProvider resource://gre/modules/HealthReport.jsm
 
 # No Aurora or Beta providers yet; use the categories
 # "healthreport-js-provider-aurora", "healthreport-js-provider-beta".
--- a/services/healthreport/healthreporter.jsm
+++ b/services/healthreport/healthreporter.jsm
@@ -424,16 +424,20 @@ AbstractHealthReporter.prototype = Objec
   /**
    * Obtain a provider from its name.
    *
    * This will only return providers that are currently initialized. If
    * a provider is lazy initialized (like pull-only providers) this
    * will likely not return anything.
    */
   getProvider: function (name) {
+    if (!this._providerManager) {
+      return null;
+    }
+
     return this._providerManager.getProvider(name);
   },
 
   _initProvider: function (provider) {
     provider.healthReporter = this;
   },
 
   /**
@@ -1148,41 +1152,54 @@ HealthReporter.prototype = Object.freeze
       };
 
       this._uploadData(request);
     }
 
     return result;
   },
 
-  _onBagheeraResult: function (request, isDelete, result) {
+  _onBagheeraResult: function (request, isDelete, date, result) {
     this._log.debug("Received Bagheera result.");
 
     let promise = CommonUtils.laterTickResolvingPromise(null);
+    let hrProvider = this.getProvider("org.mozilla.healthreport");
 
     if (!result.transportSuccess) {
+      // The built-in provider may not be initialized if this instance failed
+      // to initialize fully.
+      if (hrProvider && !isDelete) {
+        hrProvider.recordEvent("uploadTransportFailure", date);
+      }
+
       request.onSubmissionFailureSoft("Network transport error.");
       return promise;
     }
 
     if (!result.serverSuccess) {
+      if (hrProvider && !isDelete) {
+        hrProvider.recordEvent("uploadServerFailure", date);
+      }
+
       request.onSubmissionFailureHard("Server failure.");
       return promise;
     }
 
-    let now = this._now();
+    if (hrProvider && !isDelete) {
+      hrProvider.recordEvent("uploadSuccess", date);
+    }
 
     if (isDelete) {
       this.lastSubmitID = null;
     } else {
       this.lastSubmitID = result.id;
-      this.lastPingDate = now;
+      this.lastPingDate = date;
     }
 
-    request.onSubmissionSuccess(now);
+    request.onSubmissionSuccess(this._now());
 
 #ifdef PRERELEASE_BUILD
     // Intended to be temporary until we a) assess the impact b) bug 846133
     // deploys more robust storage for state.
     try {
       Services.prefs.savePrefFile(null);
     } catch (ex) {
       this._log.warn("Error forcing prefs save: " + CommonUtils.exceptionStr(ex));
@@ -1203,48 +1220,59 @@ HealthReporter.prototype = Object.freeze
   },
 
   _uploadData: function (request) {
     let id = CommonUtils.generateUUID();
 
     this._log.info("Uploading data to server: " + this.serverURI + " " +
                    this.serverNamespace + ":" + id);
     let client = new BagheeraClient(this.serverURI);
+    let now = this._now();
 
     return Task.spawn(function doUpload() {
       let payload = yield this.getJSONPayload();
 
       let histogram = Services.telemetry.getHistogramById(TELEMETRY_PAYLOAD_SIZE_UNCOMPRESSED);
       histogram.add(payload.length);
 
       TelemetryStopwatch.start(TELEMETRY_SAVE_LAST_PAYLOAD, this);
       try {
         yield this._saveLastPayload(payload);
         TelemetryStopwatch.finish(TELEMETRY_SAVE_LAST_PAYLOAD, this);
       } catch (ex) {
         TelemetryStopwatch.cancel(TELEMETRY_SAVE_LAST_PAYLOAD, this);
         throw ex;
       }
 
+      let hrProvider = this.getProvider("org.mozilla.healthreport");
+      if (hrProvider) {
+        let event = this.lastSubmitID ? "continuationUploadAttempt"
+                                      : "firstDocumentUploadAttempt";
+        hrProvider.recordEvent(event, now);
+      }
+
       TelemetryStopwatch.start(TELEMETRY_UPLOAD, this);
       let result;
       try {
         let options = {
           deleteID: this.lastSubmitID,
           telemetryCompressed: TELEMETRY_PAYLOAD_SIZE_COMPRESSED,
         };
         result = yield client.uploadJSON(this.serverNamespace, id, payload,
                                          options);
         TelemetryStopwatch.finish(TELEMETRY_UPLOAD, this);
       } catch (ex) {
         TelemetryStopwatch.cancel(TELEMETRY_UPLOAD, this);
+        if (hrProvider) {
+          hrProvider.recordEvent("uploadClientFailure", now);
+        }
         throw ex;
       }
 
-      yield this._onBagheeraResult(request, false, result);
+      yield this._onBagheeraResult(request, false, now, result);
     }.bind(this));
   },
 
   /**
    * Request deletion of remote data.
    *
    * @param request
    *        (DataSubmissionRequest) Tracks progress of this request.
@@ -1255,13 +1283,13 @@ HealthReporter.prototype = Object.freeze
       request.onNoDataAvailable();
       return;
     }
 
     this._log.warn("Deleting remote data.");
     let client = new BagheeraClient(this.serverURI);
 
     return client.deleteDocument(this.serverNamespace, this.lastSubmitID)
-                 .then(this._onBagheeraResult.bind(this, request, true),
+                 .then(this._onBagheeraResult.bind(this, request, true, this._now()),
                        this._onSubmitDataRequestFailure.bind(this));
   },
 });
 
--- a/services/healthreport/providers.jsm
+++ b/services/healthreport/providers.jsm
@@ -16,16 +16,17 @@
 
 #ifndef MERGED_COMPARTMENT
 
 this.EXPORTED_SYMBOLS = [
   "AddonsProvider",
   "AppInfoProvider",
   "CrashDirectoryService",
   "CrashesProvider",
+  "HealthReportProvider",
   "PlacesProvider",
   "SearchesProvider",
   "SessionsProvider",
   "SysInfoProvider",
 ];
 
 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
 
@@ -1389,8 +1390,46 @@ this.SearchesProvider.prototype = Object
     let id = m.interestingEngines[engine] || "other";
     let field = id + "." + source;
     return this.enqueueStorageOperation(function recordSearch() {
       return m.incrementDailyCounter(field);
     });
   },
 });
 
+function HealthReportSubmissionMeasurement1() {
+  Metrics.Measurement.call(this);
+}
+
+HealthReportSubmissionMeasurement1.prototype = Object.freeze({
+  __proto__: Metrics.Measurement.prototype,
+
+  name: "submissions",
+  version: 1,
+
+  fields: {
+    firstDocumentUploadAttempt: DAILY_COUNTER_FIELD,
+    continuationUploadAttempt: DAILY_COUNTER_FIELD,
+    uploadSuccess: DAILY_COUNTER_FIELD,
+    uploadTransportFailure: DAILY_COUNTER_FIELD,
+    uploadServerFailure: DAILY_COUNTER_FIELD,
+    uploadClientFailure: DAILY_COUNTER_FIELD,
+  },
+});
+
+this.HealthReportProvider = function () {
+  Metrics.Provider.call(this);
+}
+
+HealthReportProvider.prototype = Object.freeze({
+  __proto__: Metrics.Provider.prototype,
+
+  name: "org.mozilla.healthreport",
+
+  measurementTypes: [HealthReportSubmissionMeasurement1],
+
+  recordEvent: function (event, date=new Date()) {
+    let m = this.getMeasurement("submissions", 1);
+    return this.enqueueStorageOperation(function recordCounter() {
+      return m.incrementDailyCounter(event, date);
+    });
+  },
+});
--- a/services/healthreport/tests/xpcshell/test_healthreporter.js
+++ b/services/healthreport/tests/xpcshell/test_healthreporter.js
@@ -5,19 +5,21 @@
 
 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
 
 Cu.import("resource://services-common/observers.js");
 Cu.import("resource://gre/modules/commonjs/sdk/core/promise.js");
 Cu.import("resource://gre/modules/Metrics.jsm");
 Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://gre/modules/services/healthreport/healthreporter.jsm");
+Cu.import("resource://gre/modules/services/healthreport/providers.jsm");
 Cu.import("resource://gre/modules/services/datareporting/policy.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
+Cu.import("resource://testing-common/httpd.js");
 Cu.import("resource://testing-common/services-common/bagheeraserver.js");
 Cu.import("resource://testing-common/services/metrics/mocks.jsm");
 Cu.import("resource://testing-common/services/healthreport/utils.jsm");
 
 
 const SERVER_HOSTNAME = "localhost";
 const SERVER_PORT = 8080;
 const SERVER_URI = "http://" + SERVER_HOSTNAME + ":" + SERVER_PORT;
@@ -58,18 +60,25 @@ function getJustReporter(name, uri=SERVE
 
   let type = inspected ? InspectedHealthReporter : HealthReporter;
   reporter = new type(branch + "healthreport.", policy);
 
   return reporter;
 }
 
 function getReporter(name, uri, inspected) {
-  let reporter = getJustReporter(name, uri, inspected);
-  return reporter.onInit();
+  return Task.spawn(function init() {
+    let reporter = getJustReporter(name, uri, inspected);
+    yield reporter.onInit();
+
+    yield reporter._providerManager.registerProviderFromType(
+      HealthReportProvider);
+
+    throw new Task.Result(reporter);
+  });
 }
 
 function getReporterAndServer(name, namespace="test") {
   return Task.spawn(function get() {
     let reporter = yield getReporter(name, SERVER_URI);
     reporter.serverNamespace = namespace;
 
     let server = new BagheeraServer(SERVER_URI);
@@ -83,16 +92,36 @@ function getReporterAndServer(name, name
 
 function shutdownServer(server) {
   let deferred = Promise.defer();
   server.stop(deferred.resolve.bind(deferred));
 
   return deferred.promise;
 }
 
+function getHealthReportProviderValues(reporter, day=null) {
+  return Task.spawn(function getValues() {
+    let p = reporter.getProvider("org.mozilla.healthreport");
+    do_check_neq(p, null);
+    let m = p.getMeasurement("submissions", 1);
+    do_check_neq(m, null);
+
+    let data = yield reporter._storage.getMeasurementValues(m.id);
+    if (!day) {
+      throw new Task.Result(data);
+    }
+
+    do_check_true(data.days.hasDay(day));
+    let serializer = m.serializer(m.SERIALIZE_JSON)
+    let json = serializer.daily(data.days.getDay(day));
+
+    throw new Task.Result(json);
+  });
+}
+
 function run_test() {
   makeFakeAppDir().then(run_next_test, do_throw);
 }
 
 add_task(function test_constructor() {
   let reporter = yield getReporter("constructor");
 
   try {
@@ -184,27 +213,27 @@ add_task(function test_pull_only_provide
                       "resource://testing-common/services/metrics/mocks.jsm",
                       false, true);
   cm.addCategoryEntry(category, "DummyConstantProvider",
                       "resource://testing-common/services/metrics/mocks.jsm",
                       false, true);
 
   let reporter = yield getReporter("constant_only_providers");
   try {
-    do_check_eq(reporter._providerManager._providers.size, 0);
+    let initCount = reporter._providerManager.providers.length;
     yield reporter._providerManager.registerProvidersFromCategoryManager(category);
-    do_check_eq(reporter._providerManager._providers.size, 1);
+    do_check_eq(reporter._providerManager._providers.size, initCount + 1);
     do_check_true(reporter._storage.hasProvider("DummyProvider"));
     do_check_false(reporter._storage.hasProvider("DummyConstantProvider"));
     do_check_neq(reporter.getProvider("DummyProvider"), null);
     do_check_null(reporter.getProvider("DummyConstantProvider"));
 
     yield reporter.collectMeasurements();
 
-    do_check_eq(reporter._providerManager._providers.size, 1);
+    do_check_eq(reporter._providerManager._providers.size, initCount + 1);
     do_check_true(reporter._storage.hasProvider("DummyConstantProvider"));
 
     let mID = reporter._storage.measurementID("DummyConstantProvider", "DummyMeasurement", 1);
     let values = yield reporter._storage.getMeasurementValues(mID);
     do_check_true(values.singular.size > 0);
   } finally {
     reporter._shutdown();
   }
@@ -319,47 +348,48 @@ add_task(function test_constant_only_pro
                       "resource://testing-common/services/metrics/mocks.jsm",
                       false, true);
   cm.addCategoryEntry(category, "DummyConstantProvider",
                       "resource://testing-common/services/metrics/mocks.jsm",
                       false, true);
 
   let reporter = yield getReporter("constant_only_providers_in_json_payload");
   try {
+    let initCount = reporter._providerManager.providers.length;
     yield reporter._providerManager.registerProvidersFromCategoryManager(category);
 
     let payload = yield reporter.collectAndObtainJSONPayload();
     let o = JSON.parse(payload);
     do_check_true("DummyProvider.DummyMeasurement" in o.data.last);
     do_check_true("DummyConstantProvider.DummyMeasurement" in o.data.last);
 
     let providers = reporter._providerManager.providers;
-    do_check_eq(providers.length, 1);
+    do_check_eq(providers.length, initCount + 1);
 
     // Do it again for good measure.
     payload = yield reporter.collectAndObtainJSONPayload();
     o = JSON.parse(payload);
     do_check_true("DummyProvider.DummyMeasurement" in o.data.last);
     do_check_true("DummyConstantProvider.DummyMeasurement" in o.data.last);
 
     providers = reporter._providerManager.providers;
-    do_check_eq(providers.length, 1);
+    do_check_eq(providers.length, initCount + 1);
 
     // Ensure throwing getJSONPayload is handled properly.
     Object.defineProperty(reporter, "_getJSONPayload", {
       value: function () {
         throw new Error("Silly error.");
       },
     });
 
     let deferred = Promise.defer();
 
     reporter.collectAndObtainJSONPayload().then(do_throw, function onError() {
       providers = reporter._providerManager.providers;
-      do_check_eq(providers.length, 1);
+      do_check_eq(providers.length, initCount + 1);
       deferred.resolve();
     });
 
     yield deferred.promise;
   } finally {
     reporter._shutdown();
   }
 });
@@ -504,44 +534,85 @@ add_task(function test_data_submission_t
     reporter.serverNamespace = "test00";
 
     let deferred = Promise.defer();
     let request = new DataSubmissionRequest(deferred, new Date(Date.now + 30000));
     reporter.requestDataUpload(request);
 
     yield deferred.promise;
     do_check_eq(request.state, request.SUBMISSION_FAILURE_SOFT);
+
+    let data = yield getHealthReportProviderValues(reporter, new Date());
+    do_check_eq(data._v, 1);
+    do_check_eq(data.firstDocumentUploadAttempt, 1);
+    do_check_eq(data.uploadTransportFailure, 1);
+    do_check_eq(Object.keys(data).length, 3);
   } finally {
     reporter._shutdown();
   }
 });
 
+add_task(function test_data_submission_server_failure() {
+  let [reporter, server] = yield getReporterAndServer("data_submission_server_failure");
+  try {
+    Object.defineProperty(server, "_handleNamespaceSubmitPost", {
+      value: function (ns, id, request, response) {
+        throw HTTP_500;
+      },
+      writable: true,
+    });
+
+    let deferred = Promise.defer();
+    let now = new Date();
+    let request = new DataSubmissionRequest(deferred, now);
+    reporter.requestDataUpload(request);
+    yield deferred.promise;
+    do_check_eq(request.state, request.SUBMISSION_FAILURE_HARD);
+
+    let data = yield getHealthReportProviderValues(reporter, now);
+    do_check_eq(data._v, 1);
+    do_check_eq(data.firstDocumentUploadAttempt, 1);
+    do_check_eq(data.uploadServerFailure, 1);
+    do_check_eq(Object.keys(data).length, 3);
+  } finally {
+    yield shutdownServer(server);
+    reporter._shutdown();
+  }
+});
+
 add_task(function test_data_submission_success() {
   let [reporter, server] = yield getReporterAndServer("data_submission_success");
   try {
     yield reporter._providerManager.registerProviderFromType(DummyProvider);
     yield reporter._providerManager.registerProviderFromType(DummyConstantProvider);
 
     do_check_eq(reporter.lastPingDate.getTime(), 0);
     do_check_false(reporter.haveRemoteData());
 
     let deferred = Promise.defer();
 
-    let request = new DataSubmissionRequest(deferred, new Date());
+    let now = new Date();
+    let request = new DataSubmissionRequest(deferred, now);
     reporter.requestDataUpload(request);
     yield deferred.promise;
     do_check_eq(request.state, request.SUBMISSION_SUCCESS);
     do_check_true(reporter.lastPingDate.getTime() > 0);
     do_check_true(reporter.haveRemoteData());
 
     // Ensure data from providers made it to payload.
     let o = yield reporter.getLastPayload();
     do_check_true("DummyProvider.DummyMeasurement" in o.data.last);
     do_check_true("DummyConstantProvider.DummyMeasurement" in o.data.last);
 
+    let data = yield getHealthReportProviderValues(reporter, now);
+    do_check_eq(data._v, 1);
+    do_check_eq(data.firstDocumentUploadAttempt, 1);
+    do_check_eq(data.uploadSuccess, 1);
+    do_check_eq(Object.keys(data).length, 3);
+
     reporter._shutdown();
   } finally {
     yield shutdownServer(server);
   }
 });
 
 add_task(function test_recurring_daily_pings() {
   let [reporter, server] = yield getReporterAndServer("recurring_daily_pings");
@@ -564,16 +635,25 @@ add_task(function test_recurring_daily_p
     // Skip forward to next scheduled submission time.
     defineNow(policy, policy.nextDataSubmissionDate);
     promise = policy.checkStateAndTrigger();
     do_check_neq(promise, null);
     yield promise;
     do_check_neq(reporter.lastSubmitID, lastID);
     do_check_true(server.hasDocument(reporter.serverNamespace, reporter.lastSubmitID));
     do_check_false(server.hasDocument(reporter.serverNamespace, lastID));
+
+    // now() on the health reporter instance wasn't munged. So, we should see
+    // both requests attributed to the same day.
+    let data = yield getHealthReportProviderValues(reporter, new Date());
+    do_check_eq(data._v, 1);
+    do_check_eq(data.firstDocumentUploadAttempt, 1);
+    do_check_eq(data.continuationUploadAttempt, 1);
+    do_check_eq(data.uploadSuccess, 2);
+    do_check_eq(Object.keys(data).length, 4);
   } finally {
     reporter._shutdown();
     yield shutdownServer(server);
   }
 });
 
 add_task(function test_request_remote_data_deletion() {
   let [reporter, server] = yield getReporterAndServer("request_remote_data_deletion");
@@ -746,22 +826,22 @@ add_task(function test_upload_on_init_fa
   reporter.onInitializeProviderManagerFinished = function () {
     throw new Error("Fake error during provider manager initialization.");
   };
 
   let deferred = Promise.defer();
 
   let oldOnResult = reporter._onBagheeraResult;
   Object.defineProperty(reporter, "_onBagheeraResult", {
-    value: function (request, isDelete, result) {
+    value: function (request, isDelete, date, result) {
       do_check_false(isDelete);
       do_check_true(result.transportSuccess);
       do_check_true(result.serverSuccess);
 
-      oldOnResult.call(reporter, request, isDelete, result);
+      oldOnResult.call(reporter, request, isDelete, new Date(), result);
       deferred.resolve();
     },
   });
 
   reporter._policy.recordUserAcceptance();
   let error = false;
   try {
     yield reporter.onInit();