author | Jan de Mooij <jdemooij@mozilla.com> |
Wed, 27 Mar 2013 10:28:41 +0100 | |
changeset 127448 | 8a80ce1f0c2201b3267c866943a1095abfba3d23 |
parent 127447 | 607291e864e383c9202cebfe1c93c743ba4a8a7c (current diff) |
parent 126236 | 178a4a770bb1664d431eb0428f057bb48f0ad1ba (diff) |
child 127449 | 6cac55064722597d0292f530678fdb0c21adf650 |
push id | 24503 |
push user | jandemooij@gmail.com |
push date | Wed, 03 Apr 2013 15:43:00 +0000 |
treeherder | mozilla-central@b5cb88ccd907 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
milestone | 22.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/browser/base/content/abouthealthreport/abouthealth.css +++ b/browser/base/content/abouthealthreport/abouthealth.css @@ -2,139 +2,14 @@ margin: 0; padding: 0; } html, body { height: 100%; } -body { - background-color: window; - color: windowtext; - font-family: "Trebuchet MS", "Helvetica"; -} - -#about-header { - padding: 6px 20px; - min-height: 60px; - border-bottom: 1px solid #999999; - margin-bottom: 12px; - display: flex; - align-items: center; - justify-content: space-between; - background-image: linear-gradient(to bottom, #E66000, #BB2200); - color: #FFFFFF; -} - -#control-container { - display: flex; - align-items: center; -} - -#content-line { - display: flex; - justify-content: space-between; - align-items: center; -} - -#content { - display: flex; - flex-direction: column; -} - -#state-intro { - background-image: linear-gradient(to bottom, #EAEFF2, #D4DDE4); - border: 1px solid #999999; - border-radius: 6px; - margin: 12px; - padding: 12px; -} - -#settings-controls { - padding-top: 15px; -} - -#control-container { - padding-top: 15px; -} - -#content[state="default"] #details-hide, -#content[state="default"] #btn-optin, -#content[state="default"] #intro-disabled { - display: none; -} - -#content[state="showDetails"] #details-show, -#content[state="showDetails"] #btn-optin, -#content[state="showDetails"] #intro-disabled { - display: none; -} - -#content[state="showReport"] #details-hide, -#content[state="showReport"] #report-show, -#content[state="showReport"] #btn-optin, -#content[state="showReport"] #intro-disabled { - display: none; -} - -#content[state="disabled"] #details-hide, -#content[state="disabled"] #details-show, -#content[state="disabled"] #btn-optout, -#content[state="disabled"] #intro-enabled { - display: none; -} - -#details-view, -#report-view { - display: none; -} - -#data-view { - height: auto; - margin-top: 8px; - align-items: center; - justify-content: center; - border: 1px solid #999999; - border-radius: 6px; - margin: 12px; -} - -#remote-report, -#report-view { +#remote-report { width: 100%; height: 100%; - min-height: 600px; -} - -#report-show { + border: 0; display: flex; - width: 100%; - height: 100%; - min-height: 60px; - font-size: 18px; - font-weight: bold; - background-image: linear-gradient(to bottom, #80BB2E, #547D1C); - color: #ffffff; - border-radius: 6px; } - -#details-view { - border: 1px solid #999999; - border-radius: 6px; - margin: 12px; - padding: 12px; -} - -#content[state="showDetails"], -#content[state="showReport"], -#content[state="showDetails"] #details-view, -#content[state="showReport"] #report-view { - display: block; -} - -#content[state="showReport"] #report-show { - display: none; -} -#content[state="showReport"] #report-view, -#remote-report { - border: 0; -}
--- a/browser/base/content/abouthealthreport/abouthealth.js +++ b/browser/base/content/abouthealthreport/abouthealth.js @@ -2,115 +2,140 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ "use strict"; const {classes: Cc, interfaces: Ci, utils: Cu} = Components; Cu.import("resource://services-common/preferences.js"); +Cu.import("resource://gre/modules/Services.jsm"); const reporter = Cc["@mozilla.org/datareporting/service;1"] .getService(Ci.nsISupports) .wrappedJSObject .healthReporter; const policy = Cc["@mozilla.org/datareporting/service;1"] .getService(Ci.nsISupports) .wrappedJSObject .policy; -const prefs = new Preferences("datareporting.healthreport.about."); +const prefs = new Preferences("datareporting.healthreport."); + + +let healthReportWrapper = { + init: function () { + reporter.onInit().then(healthReportWrapper.refreshPayload, + healthReportWrapper.handleInitFailure); -function getLocale() { - return Cc["@mozilla.org/chrome/chrome-registry;1"] - .getService(Ci.nsIXULChromeRegistry) - .getSelectedLocale("global"); -} + let iframe = document.getElementById("remote-report"); + iframe.addEventListener("load", healthReportWrapper.initRemotePage, false); + let report = this._getReportURI(); + iframe.src = report.spec; + prefs.observe("uploadEnabled", this.updatePrefState, healthReportWrapper); + }, -function init() { - refreshWithDataSubmissionFlag(policy.healthReportUploadEnabled); - refreshJSONPayload(); - document.getElementById("details-link").href = prefs.get("glossaryUrl"); -} + uninit: function () { + prefs.ignore("uploadEnabled", this.updatePrefState, healthReportWrapper); + }, + + _getReportURI: function () { + let url = Services.urlFormatter.formatURLPref("datareporting.healthreport.about.reportUrl"); + return Services.io.newURI(url, null, null); + }, -/** - * Update the state of the page to reflect the current data submission state. - * - * @param enabled - * (bool) Whether data submission is enabled. - */ -function refreshWithDataSubmissionFlag(enabled) { - if (!enabled) { - updateView("disabled"); - } else { - updateView("default"); - } -} + onOptIn: function () { + policy.recordHealthReportUploadEnabled(true, + "Health report page sent opt-in command."); + this.updatePrefState(); + }, + + onOptOut: function () { + policy.recordHealthReportUploadEnabled(false, + "Health report page sent opt-out command."); + this.updatePrefState(); + }, -function updateView(state="default") { - let content = document.getElementById("content"); - let controlContainer = document.getElementById("control-container"); - content.setAttribute("state", state); - controlContainer.setAttribute("state", state); -} + updatePrefState: function () { + try { + let prefs = { + enabled: policy.healthReportUploadEnabled, + } + this.injectData("prefs", prefs); + } catch (e) { + this.reportFailure(this.ERROR_PREFS_FAILED); + } + }, -function refreshDataView(data) { - let noData = document.getElementById("data-no-data"); - let dataEl = document.getElementById("raw-data"); + refreshPayload: function () { + reporter.collectAndObtainJSONPayload().then(healthReportWrapper.updatePayload, + healthReportWrapper.handlePayloadFailure); + }, - noData.style.display = data ? "none" : "inline"; - dataEl.style.display = data ? "block" : "none"; - if (data) { - dataEl.textContent = JSON.stringify(data, null, 2); - } -} + updatePayload: function (data) { + healthReportWrapper.injectData("payload", data); + }, -/** - * Ensure the page has the latest version of the uploaded JSON payload. - */ -function refreshJSONPayload() { - reporter.getLastPayload().then(refreshDataView); -} + injectData: function (type, content) { + let report = this._getReportURI(); + + // file URIs can't be used for targetOrigin, so we use "*" for this special case + // in all other cases, pass in the URL to the report so we properly restrict the message dispatch + let reportUrl = report.scheme == "file" ? "*" : report.spec; -function onOptInClick() { - policy.recordHealthReportUploadEnabled(true, - "Clicked opt in button on about page."); - refreshWithDataSubmissionFlag(true); -} + let data = { + type: type, + content: content + } -function onOptOutClick() { - let prompts = Cc["@mozilla.org/embedcomp/prompt-service;1"] - .getService(Ci.nsIPromptService); - - let messages = document.getElementById("optout-confirmationPrompt"); - let title = messages.getAttribute("confirmationPrompt_title"); - let message = messages.getAttribute("confirmationPrompt_message"); - - if (!prompts.confirm(window, title, message)) { - return; - } + let iframe = document.getElementById("remote-report"); + iframe.contentWindow.postMessage(data, reportUrl); + }, - policy.recordHealthReportUploadEnabled(false, - "Clicked opt out button on about page."); - refreshWithDataSubmissionFlag(false); - updateView("disabled"); -} - -function onShowRawDataClick() { - updateView("showDetails"); - refreshJSONPayload(); -} + handleRemoteCommand: function (evt) { + switch (evt.detail.command) { + case "DisableDataSubmission": + this.onOptOut(); + break; + case "EnableDataSubmission": + this.onOptIn(); + break; + case "RequestCurrentPrefs": + this.updatePrefState(); + break; + case "RequestCurrentPayload": + this.refreshPayload(); + break; + default: + Cu.reportError("Unexpected remote command received: " + evt.detail.command + ". Ignoring command."); + break; + } + }, -function onHideRawDataClick() { - updateView("default"); -} + initRemotePage: function () { + let iframe = document.getElementById("remote-report").contentDocument; + iframe.addEventListener("RemoteHealthReportCommand", + function onCommand(e) {healthReportWrapper.handleRemoteCommand(e);}, + false); + healthReportWrapper.updatePrefState(); + }, + + // error handling + ERROR_INIT_FAILED: 1, + ERROR_PAYLOAD_FAILED: 2, + ERROR_PREFS_FAILED: 3, -function onShowReportClick() { - updateView("showReport"); - document.getElementById("remote-report").src = prefs.get("reportUrl"); -} + reportFailure: function (error) { + let details = { + errorType: error, + } + healthReportWrapper.injectData("error", details); + }, -function onHideReportClick() { - updateView("default"); - document.getElementById("remote-report").src = ""; + handleInitFailure: function () { + healthReportWrapper.reportFailure(healthReportWrapper.ERROR_INIT_FAILED); + }, + + handlePayloadFailure: function () { + healthReportWrapper.reportFailure(healthReportWrapper.ERROR_PAYLOAD_FAILED); + }, } -
--- a/browser/base/content/abouthealthreport/abouthealth.xhtml +++ b/browser/base/content/abouthealthreport/abouthealth.xhtml @@ -17,55 +17,14 @@ <head> <title>&abouthealth.pagetitle;</title> <link rel="stylesheet" href="chrome://browser/content/abouthealthreport/abouthealth.css" type="text/css" /> <script type="text/javascript;version=1.8" src="chrome://browser/content/abouthealthreport/abouthealth.js" /> </head> - <body dir="&abouthealth.locale-direction;" class="aboutPageWideContainer" onload="init();"> - <div id="about-header"> - <h1>&abouthealth.header;</h1> - <img src="chrome://branding/content/icon48.png"/> - </div> - - <div id="content"> - <div id="state-intro"> - <h3>&abouthealth.intro.title;</h3> - <div id="content-line"> - <p id="intro-enabled">&abouthealth.intro-enabled;</p> - <p id="intro-disabled">&abouthealth.intro-disabled;</p> - <div id="control-container"> - <button id="btn-optin" onclick="onOptInClick();">&abouthealth.optin;</button> - <button id="btn-optout" onclick="onOptOutClick();">&abouthealth.optout;</button> - - <button id="details-show" onclick="onShowRawDataClick()">&abouthealth.show-raw-data;</button> - <button id="details-hide" onclick="onHideRawDataClick()">&abouthealth.hide-raw-data;</button> - </div> - </div> - </div> - <div id="details-view"> - <p id="details-description"> - &abouthealth.details.description-start; - <a id="details-link">&abouthealth.details-link;</a> - &abouthealth.details.description-end; - </p> - - <p id="data-no-data">&abouthealth.no-data-available;</p> - <pre id="raw-data" style="display: none"></pre> - </div> - - <div id="data-view"> - <button id="report-show" onclick="onShowReportClick()">&abouthealth.show-report;</button> - <div id="report-view"> - <iframe id="remote-report"/> - </div> - </div> - </div> - - <input type="hidden" id="optout-confirmationPrompt" - confirmationPrompt_title="&abouthealth.optout.confirmationPrompt.title;" - confirmationPrompt_message="&abouthealth.optout.confirmationPrompt.message;" - /> + <body onload="healthReportWrapper.init();" + onunload="healthReportWrapper.uninit();"> + <iframe id="remote-report"/> </body> </html>
--- a/browser/base/content/test/Makefile.in +++ b/browser/base/content/test/Makefile.in @@ -311,17 +311,19 @@ endif browser_URLBarSetURI.js \ browser_bookmark_titles.js \ browser_pageInfo_plugins.js \ browser_pageInfo.js \ feed_tab.html \ browser_pluginCrashCommentAndURL.js \ pluginCrashCommentAndURL.html \ browser_private_no_prompt.js \ - browser_blob-channelname.js + browser_blob-channelname.js \ + browser_aboutHealthReport.js \ + healthreport_testRemoteCommands.html \ $(NULL) # Disable test on Windows due to frequent failures (bug 841341) ifneq (windows,$(MOZ_WIDGET_TOOLKIT)) _BROWSER_FILES += \ browser_popupNotification.js \ $(NULL) endif
new file mode 100644 --- /dev/null +++ b/browser/base/content/test/browser_aboutHealthReport.js @@ -0,0 +1,105 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +XPCOMUtils.defineLazyModuleGetter(this, "Promise", + "resource://gre/modules/commonjs/sdk/core/promise.js"); +XPCOMUtils.defineLazyModuleGetter(this, "Task", + "resource://gre/modules/Task.jsm"); + +registerCleanupFunction(function() { + // Ensure we don't pollute prefs for next tests. + try { + Services.prefs.clearUserPref("datareporting.healthreport.about.reportUrl"); + let policy = Cc["@mozilla.org/datareporting/service;1"] + .getService(Ci.nsISupports) + .wrappedJSObject + .policy; + policy.recordHealthReportUploadEnabled(true, + "Resetting after tests."); + } catch (ex) {} +}); + +let gTests = [ + +{ + desc: "Test the remote commands", + setup: function () + { + Services.prefs.setCharPref("datareporting.healthreport.about.reportUrl", + "https://example.com/browser/browser/base/content/test/healthreport_testRemoteCommands.html"); + }, + run: function () + { + let deferred = Promise.defer(); + + let policy = Cc["@mozilla.org/datareporting/service;1"] + .getService(Ci.nsISupports) + .wrappedJSObject + .policy; + + let results = 0; + try { + let win = gBrowser.contentWindow; + win.addEventListener("message", function testLoad(e) { + if (e.data.type == "testResult") { + ok(e.data.pass, e.data.info); + results++; + } + else if (e.data.type == "testsComplete") { + is(results, e.data.count, "Checking number of results received matches the number of tests that should have run"); + win.removeEventListener("message", testLoad, false, true); + deferred.resolve(); + } + + }, false, true); + + } catch(e) { + ok(false, "Failed to get all commands"); + deferred.reject(); + } + return deferred.promise; + } +}, + + +]; // gTests + +function test() +{ + waitForExplicitFinish(); + + // xxxmpc leaving this here until we resolve bug 854038 and bug 854060 + requestLongerTimeout(10); + + Task.spawn(function () { + for (let test of gTests) { + info(test.desc); + test.setup(); + + yield promiseNewTabLoadEvent("about:healthreport"); + + yield test.run(); + + gBrowser.removeCurrentTab(); + } + + finish(); + }); +} + +function promiseNewTabLoadEvent(aUrl, aEventType="load") +{ + let deferred = Promise.defer(); + let tab = gBrowser.selectedTab = gBrowser.addTab(aUrl); + tab.linkedBrowser.addEventListener(aEventType, function load(event) { + tab.linkedBrowser.removeEventListener(aEventType, load, true); + let iframe = tab.linkedBrowser.contentDocument.getElementById("remote-report"); + iframe.addEventListener("load", function frameLoad(e) { + iframe.removeEventListener("load", frameLoad, false); + deferred.resolve(); + }, false); + }, true); + return deferred.promise; +} +
new file mode 100644 --- /dev/null +++ b/browser/base/content/test/healthreport_testRemoteCommands.html @@ -0,0 +1,128 @@ +<html> + <head> + <meta charset="utf-8"> + +<script> + +function init() { + window.addEventListener("message", function process(e) {doTest(e)}, false); + doTest(); +} + +function checkSubmissionValue(payload, expectedValue) { + return payload.enabled == expectedValue; +} + +function validatePayload(payload) { + payload = JSON.parse(payload); + + // xxxmpc - this is some pretty low-bar validation, but we have plenty of tests of that API elsewhere + if (!payload.thisPingDate) + return false; + + return true; +} + +var tests = [ +{ + info: "Checking initial value is enabled", + event: "RequestCurrentPrefs", + payloadType: "prefs", + validateResponse: function(payload) { + return checkSubmissionValue(payload, true); + }, +}, +{ + info: "Verifying disabling works", + event: "DisableDataSubmission", + payloadType: "prefs", + validateResponse: function(payload) { + return checkSubmissionValue(payload, false); + }, +}, +{ + info: "Verifying we're still disabled", + event: "RequestCurrentPrefs", + payloadType: "prefs", + validateResponse: function(payload) { + return checkSubmissionValue(payload, false); + }, +}, +{ + info: "Verifying we can get a payload while submission is disabled", + event: "RequestCurrentPayload", + payloadType: "payload", + validateResponse: function(payload) { + return validatePayload(payload); + }, +}, +{ + info: "Verifying enabling works", + event: "EnableDataSubmission", + payloadType: "prefs", + validateResponse: function(payload) { + return checkSubmissionValue(payload, true); + }, +}, +{ + info: "Verifying we're still re-enabled", + event: "RequestCurrentPrefs", + payloadType: "prefs", + validateResponse: function(payload) { + return checkSubmissionValue(payload, true); + }, +}, +{ + info: "Verifying we can get a payload after re-enabling", + event: "RequestCurrentPayload", + payloadType: "payload", + validateResponse: function(payload) { + return validatePayload(payload); + }, +}, +]; + +var currentTest = -1; +function doTest(evt) { + if (evt) { + if (currentTest < 0 || !evt.data.content) + return; // not yet testing + + var test = tests[currentTest]; + if (evt.data.type != test.payloadType) + return; // skip unrequested events + + var error = JSON.stringify(evt.data.content); + var pass = false; + try { + pass = test.validateResponse(evt.data.content) + } catch (e) {} + reportResult(test.info, pass, error); + } + // start the next test if there are any left + if (tests[++currentTest]) + sendToBrowser(tests[currentTest].event); + else + reportFinished(); +} + +function reportResult(info, pass, error) { + var data = {type: "testResult", info: info, pass: pass, error: error}; + window.parent.postMessage(data, "*"); +} + +function reportFinished(cmd) { + var data = {type: "testsComplete", count: tests.length}; + window.parent.postMessage(data, "*"); +} + +function sendToBrowser(type) { + var event = new CustomEvent("RemoteHealthReportCommand", {detail: {command: type}, bubbles: true}); + document.dispatchEvent(event); +} + +</script> + </head> + <body onload="init()"> + </body> +</html>
--- a/browser/locales/en-US/chrome/browser/aboutHealthReport.dtd +++ b/browser/locales/en-US/chrome/browser/aboutHealthReport.dtd @@ -1,31 +1,6 @@ <!-- This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> -<!-- metrics.locale-direction instead of the usual local.dir entity, so RTL can skip translating page. --> -<!ENTITY abouthealth.locale-direction "ltr"> - <!-- LOCALIZATION NOTE (abouthealth.pagetitle): Firefox Health Report is a proper noun in en-US, please keep this in mind. --> <!ENTITY abouthealth.pagetitle "&brandShortName; Health Report"> -<!ENTITY abouthealth.header "&brandFullName; Health Report"> - -<!ENTITY abouthealth.intro.title "What is &brandShortName; Health Report?"> - -<!ENTITY abouthealth.intro-enabled "&brandFullName; collects some data about your computer and usage in order to provide you with a better browser experience."> -<!ENTITY abouthealth.intro-disabled "You are currently not submitting usage data to &vendorShortName;. You can help us make &brandShortName; better by clicking the "&abouthealth.optin;" button."> - -<!ENTITY abouthealth.optin "Help make &brandShortName; better"> -<!ENTITY abouthealth.optout "Turn Off Reporting"> - -<!ENTITY abouthealth.optout.confirmationPrompt.title "Stop data submission?"> -<!ENTITY abouthealth.optout.confirmationPrompt.message "Are you sure you want to opt out and delete all your anonymous product data stored on &vendorShortName; servers?"> - -<!ENTITY abouthealth.show-raw-data "Show Details"> -<!ENTITY abouthealth.hide-raw-data "Hide Details"> - -<!ENTITY abouthealth.show-report "Show &brandShortName; Report"> - -<!ENTITY abouthealth.details.description-start "This is the data &brandFullName; is submitting in order for &brandShortName; Health Report to function. You can "> -<!ENTITY abouthealth.details-link "learn more"> -<!ENTITY abouthealth.details.description-end " about what we collect and submit."> -<!ENTITY abouthealth.no-data-available "There is no previous submission to display. Please check back later.">
--- a/netwerk/base/public/nsIBrowserSearchService.idl +++ b/netwerk/base/public/nsIBrowserSearchService.idl @@ -17,17 +17,17 @@ interface nsISearchSubmission : nsISuppo readonly attribute nsIInputStream postData; /** * The URI to submit a search to. */ readonly attribute nsIURI uri; }; -[scriptable, uuid(6839f025-2e25-408e-892e-c2c2fa5650c5)] +[scriptable, uuid(ccf6aa20-10a9-4a0c-a81d-31b10ea846de)] interface nsISearchEngine : nsISupports { /** * Gets a nsISearchSubmission object that contains information about what to * send to the search engine, including the URI and postData, if applicable. * * @param data * Data to add to the submission object. @@ -127,16 +127,22 @@ interface nsISearchEngine : nsISupports */ readonly attribute AString searchForm; /** * The search engine type. */ readonly attribute long type; + /** + * An optional unique identifier for this search engine within the context of + * the distribution, as provided by the distributing entity. + */ + readonly attribute AString identifier; + }; /** * Callback for asynchronous initialization of nsIBrowserSearchService */ [scriptable, function, uuid(02256156-16e4-47f1-9979-76ff98ceb590)] interface nsIBrowserSearchInitObserver : nsISupports {
--- a/services/datareporting/tests/xpcshell/test_policy.js +++ b/services/datareporting/tests/xpcshell/test_policy.js @@ -687,58 +687,68 @@ add_test(function test_polling_implicit_ value: 250, }); Object.defineProperty(policy, "IMPLICIT_ACCEPTANCE_INTERVAL_MSEC", { value: 750, }); let count = 0; + + // Track JS elapsed time, so we can decide if we've waited for enough ticks. + let start; Object.defineProperty(policy, "checkStateAndTrigger", { value: function CheckStateAndTriggerProxy() { count++; - print("checkStateAndTrigger count: " + count); + let now = Date.now(); + let delta = now - start; + print("checkStateAndTrigger count: " + count + ", now " + now + + ", delta " + delta); // Account for some slack. DataReportingPolicy.prototype.checkStateAndTrigger.call(policy); // What should happen on different invocations: // // 1) We are inside the prompt interval so user gets prompted. // 2) still ~300ms away from implicit acceptance // 3) still ~50ms away from implicit acceptance // 4) Implicit acceptance recorded. Data submission requested. // 5) Request still pending. No new submission requested. + // + // Note that, due to the inaccuracy of timers, 4 might not happen until 5 + // firings have occurred. Yay. So we watch times, not just counts. do_check_eq(listener.notifyUserCount, 1); if (count == 1) { listener.lastNotifyRequest.onUserNotifyComplete(); } - if (count < 4) { + if (delta <= 750) { do_check_false(policy.dataSubmissionPolicyAccepted); do_check_eq(listener.requestDataUploadCount, 0); - } else { + } else if (count > 3) { do_check_true(policy.dataSubmissionPolicyAccepted); do_check_eq(policy.dataSubmissionPolicyResponseType, "accepted-implicit-time-elapsed"); do_check_eq(listener.requestDataUploadCount, 1); } - if (count > 4) { + if ((count > 4) && policy.dataSubmissionPolicyAccepted) { do_check_eq(listener.requestDataUploadCount, 1); policy.stopPolling(); run_next_test(); } } }); policy.firstRunDate = new Date(Date.now() - 4 * 24 * 60 * 60 * 1000); policy.nextDataSubmissionDate = new Date(Date.now()); + start = Date.now(); policy.startPolling(); }); add_test(function test_record_health_report_upload_enabled() { let [policy, policyPrefs, hrPrefs, listener] = getPolicy("record_health_report_upload_enabled"); // Preconditions. do_check_false(policy.pendingDeleteRemoteData);
--- a/services/healthreport/HealthReportComponents.manifest +++ b/services/healthreport/HealthReportComponents.manifest @@ -1,10 +1,12 @@ # Register Firefox Health Report providers. -category healthreport-js-provider AddonsProvider resource://gre/modules/HealthReport.jsm -category healthreport-js-provider AppInfoProvider resource://gre/modules/HealthReport.jsm -category healthreport-js-provider CrashesProvider resource://gre/modules/HealthReport.jsm -category healthreport-js-provider SysInfoProvider resource://gre/modules/HealthReport.jsm -category healthreport-js-provider ProfileMetadataProvider resource://gre/modules/HealthReport.jsm -category healthreport-js-provider SearchesProvider resource://gre/modules/HealthReport.jsm -category healthreport-js-provider SessionsProvider resource://gre/modules/HealthReport.jsm -category healthreport-js-provider PlacesProvider resource://gre/modules/HealthReport.jsm +category healthreport-js-provider-default AddonsProvider resource://gre/modules/HealthReport.jsm +category healthreport-js-provider-default AppInfoProvider resource://gre/modules/HealthReport.jsm +category healthreport-js-provider-default CrashesProvider resource://gre/modules/HealthReport.jsm +category healthreport-js-provider-default PlacesProvider resource://gre/modules/HealthReport.jsm +category healthreport-js-provider-default ProfileMetadataProvider resource://gre/modules/HealthReport.jsm +category healthreport-js-provider-default SearchesProvider resource://gre/modules/HealthReport.jsm +category healthreport-js-provider-default SessionsProvider resource://gre/modules/HealthReport.jsm +category healthreport-js-provider-default SysInfoProvider resource://gre/modules/HealthReport.jsm +# No Aurora or Beta providers yet; use the categories +# "healthreport-js-provider-aurora", "healthreport-js-provider-beta".
--- a/services/healthreport/healthreport-prefs.js +++ b/services/healthreport/healthreport-prefs.js @@ -15,12 +15,18 @@ pref("datareporting.healthreport.nextDat pref("datareporting.healthreport.pendingDeleteRemoteData", false); // Health Report is enabled by default on all channels. pref("datareporting.healthreport.uploadEnabled", true); pref("datareporting.healthreport.service.enabled", true); pref("datareporting.healthreport.service.loadDelayMsec", 10000); pref("datareporting.healthreport.service.loadDelayFirstRunMsec", 60000); -pref("datareporting.healthreport.service.providerCategories", "healthreport-js-provider"); -pref("datareporting.healthreport.about.glossaryUrl", "https://services.mozilla.com/healthreport/glossary.html"); -pref("datareporting.healthreport.about.reportUrl", "https://services.mozilla.com/healthreport/placeholder.html"); +pref("datareporting.healthreport.service.providerCategories", +#if MOZ_UPDATE_CHANNEL == release + "healthreport-js-provider-default" +#else + "healthreport-js-provider-default,healthreport-js-provider-@MOZ_UPDATE_CHANNEL@" +#endif + ); + +pref("datareporting.healthreport.about.reportUrl", "https://fhr.cdn.mozilla.net/%LOCALE%/");
--- a/services/healthreport/healthreporter.jsm +++ b/services/healthreport/healthreporter.jsm @@ -48,16 +48,17 @@ const TELEMETRY_JSON_PAYLOAD_SERIALIZE = const TELEMETRY_PAYLOAD_SIZE_UNCOMPRESSED = "HEALTHREPORT_PAYLOAD_UNCOMPRESSED_BYTES"; const TELEMETRY_PAYLOAD_SIZE_COMPRESSED = "HEALTHREPORT_PAYLOAD_COMPRESSED_BYTES"; const TELEMETRY_SAVE_LAST_PAYLOAD = "HEALTHREPORT_SAVE_LAST_PAYLOAD_MS"; const TELEMETRY_UPLOAD = "HEALTHREPORT_UPLOAD_MS"; const TELEMETRY_SHUTDOWN_DELAY = "HEALTHREPORT_SHUTDOWN_DELAY_MS"; const TELEMETRY_COLLECT_CONSTANT = "HEALTHREPORT_COLLECT_CONSTANT_DATA_MS"; const TELEMETRY_COLLECT_DAILY = "HEALTHREPORT_COLLECT_DAILY_MS"; const TELEMETRY_SHUTDOWN = "HEALTHREPORT_SHUTDOWN_MS"; +const TELEMETRY_COLLECT_CHECKPOINT = "HEALTHREPORT_POST_COLLECT_CHECKPOINT_MS"; /** * This is the abstract base class of `HealthReporter`. It exists so that * we can sanely divide work on platforms where control of Firefox Health * Report is outside of Gecko (e.g., Android). */ function AbstractHealthReporter(branch, policy, sessionRecorder) { if (!branch.endsWith(".")) { @@ -529,16 +530,28 @@ AbstractHealthReporter.prototype = Objec TelemetryStopwatch.finish(TELEMETRY_COLLECT_DAILY, this); } catch (ex) { TelemetryStopwatch.cancel(TELEMETRY_COLLECT_DAILY, this); this._log.warn("Error collecting daily data from providers: " + CommonUtils.exceptionStr(ex)); } } + // Flush gathered data to disk. This will incur an fsync. But, if + // there is ever a time we want to persist data to disk, it's + // after a massive collection. + try { + TelemetryStopwatch.start(TELEMETRY_COLLECT_CHECKPOINT, this); + yield this._storage.checkpoint(); + TelemetryStopwatch.finish(TELEMETRY_COLLECT_CHECKPOINT, this); + } catch (ex) { + TelemetryStopwatch.cancel(TELEMETRY_COLLECT_CHECKPOINT, this); + throw ex; + } + throw new Task.Result(); }.bind(this)); }, /** * Helper function to perform data collection and obtain the JSON payload. * * If you are looking for an up-to-date snapshot of FHR data that pulls in
--- a/services/healthreport/providers.jsm +++ b/services/healthreport/providers.jsm @@ -274,19 +274,17 @@ AppInfoProvider.prototype = Object.freez this._log.info("Recording new platform build ID: " + build); let m = this.getMeasurement("versions", 2); m.addDailyDiscreteText("platformBuildID", build); return this.setState("lastPlatformBuildID", build); }, collectConstantData: function () { - return this.enqueueStorageOperation(function collect() { - return Task.spawn(this._populateConstants.bind(this)); - }.bind(this)); + return this.storage.enqueueTransaction(this._populateConstants.bind(this)); }, _populateConstants: function () { let m = this.getMeasurement(AppInfoMeasurement.prototype.name, AppInfoMeasurement.prototype.version); let ai; try { @@ -419,19 +417,17 @@ SysInfoProvider.prototype = Object.freez device: "device", hardware: "hardware", name: "name", version: "version", arch: "architecture", }, collectConstantData: function () { - return this.enqueueStorageOperation(function collection() { - return Task.spawn(this._populateConstants.bind(this)); - }.bind(this)); + return this.storage.enqueueTransaction(this._populateConstants.bind(this)); }, _populateConstants: function () { let m = this.getMeasurement(SysInfoMeasurement.prototype.name, SysInfoMeasurement.prototype.version); let si = Cc["@mozilla.org/system-info;1"] .getService(Ci.nsIPropertyBag2); @@ -873,33 +869,37 @@ CrashesProvider.prototype = Object.freez name: "org.mozilla.crashes", measurementTypes: [DailyCrashesMeasurement], pullOnly: true, collectConstantData: function () { - return Task.spawn(this._populateCrashCounts.bind(this)); + return this.storage.enqueueTransaction(this._populateCrashCounts.bind(this)); }, _populateCrashCounts: function () { let now = new Date(); let service = new CrashDirectoryService(); let pending = yield service.getPendingFiles(); let submitted = yield service.getSubmittedFiles(); + function getAgeLimit() { + return 0; + } + let lastCheck = yield this.getState("lastCheck"); if (!lastCheck) { - lastCheck = 0; + lastCheck = getAgeLimit(); } else { lastCheck = parseInt(lastCheck, 10); if (Number.isNaN(lastCheck)) { - lastCheck = 0; + lastCheck = getAgeLimit(); } } let m = this.getMeasurement("crashes", 1); // FUTURE detect mtimes in the future and react more intelligently. for (let filename in pending) { let modified = pending[filename].modified; @@ -1066,25 +1066,21 @@ PlacesProvider.prototype = Object.freeze PlacesDBUtils.telemetry(null, function onResult(data) { deferred.resolve(data); }); return deferred.promise; }, }); - -/** - * Records search counts per day per engine and where search initiated. - */ -function SearchCountMeasurement() { +function SearchCountMeasurement1() { Metrics.Measurement.call(this); } -SearchCountMeasurement.prototype = Object.freeze({ +SearchCountMeasurement1.prototype = Object.freeze({ __proto__: Metrics.Measurement.prototype, name: "counts", version: 1, // We only record searches for search engines that have partner agreements // with Mozilla. fields: { @@ -1104,24 +1100,216 @@ SearchCountMeasurement.prototype = Objec "yahoo.contextmenu": DAILY_COUNTER_FIELD, "yahoo.searchbar": DAILY_COUNTER_FIELD, "yahoo.urlbar": DAILY_COUNTER_FIELD, "other.abouthome": DAILY_COUNTER_FIELD, "other.contextmenu": DAILY_COUNTER_FIELD, "other.searchbar": DAILY_COUNTER_FIELD, "other.urlbar": DAILY_COUNTER_FIELD, }, +}); - // If an engine is removed from this list, it may not be reported any more. - // Verify side-effects are sane before removing an entry. - PARTNER_ENGINES: [ - "amazon.com", +/** + * Records search counts per day per engine and where search initiated. + * + * We want to record granular details for individual locale-specific search + * providers, but only if they're Mozilla partners. In order to do this, we + * track the nsISearchEngine identifier, which denotes shipped search engines, + * and intersect those with our partner list. + * + * We don't use the search engine name directly, because it is shared across + * locales; e.g., eBay-de and eBay both share the name "eBay". + */ +function SearchCountMeasurement2() { + this._fieldSpecs = null; + this._interestingEngines = null; // Name -> ID. ("Amazon.com" -> "amazondotcom") + + Metrics.Measurement.call(this); +} + +SearchCountMeasurement2.prototype = Object.freeze({ + __proto__: Metrics.Measurement.prototype, + + name: "counts", + version: 2, + + /** + * Default implementation; can be overridden by test helpers. + */ + getDefaultEngines: function () { + return Services.search.getDefaultEngines(); + }, + + _initialize: function () { + // Don't create all of these for every profile. + // There are 61 partner engines, translating to 244 fields. + // Instead, compute only those that are possible -- those for whom the + // provider is one of the default search engines. + // This set can grow over time, and change as users run different localized + // Firefox instances. + this._fieldSpecs = {}; + this._interestingEngines = {}; + + for (let source of this.SOURCES) { + this._fieldSpecs["other." + source] = DAILY_COUNTER_FIELD; + } + + let engines = this.getDefaultEngines(); + for (let engine of engines) { + let id = engine.identifier; + if (!id || (this.PROVIDERS.indexOf(id) == -1)) { + continue; + } + + this._interestingEngines[engine.name] = id; + let fieldPrefix = id + "."; + for (let source of this.SOURCES) { + this._fieldSpecs[fieldPrefix + source] = DAILY_COUNTER_FIELD; + } + } + }, + + // Our fields are dynamic, so we compute them into _fieldSpecs by looking at + // the current set of interesting engines. + get fields() { + if (!this._fieldSpecs) { + this._initialize(); + } + return this._fieldSpecs; + }, + + get interestingEngines() { + if (!this._fieldSpecs) { + this._initialize(); + } + return this._interestingEngines; + }, + + /** + * Override the default behavior: serializers should include every counter + * field from the DB, even if we don't currently have it registered. + * + * Do this so we don't have to register several hundred fields to match + * various Firefox locales. + * + * We use the "provider.type" syntax as a rudimentary check for validity. + * + * We trust that measurement versioning is sufficient to exclude old provider + * data. + */ + shouldIncludeField: function (name) { + return name.indexOf(".") != -1; + }, + + /** + * The measurement type mechanism doesn't introspect the DB. Override it + * so that we can assume all unknown fields are counters. + */ + fieldType: function (name) { + if (name in this.fields) { + return this.fields[name].type; + } + + // Default to a counter. + return Metrics.Storage.FIELD_DAILY_COUNTER; + }, + + // You can compute the total list of fields by unifying the entire l10n repo + // set with the list of partners: + // + // sort -u */*/searchplugins/list.txt | tr -d '^M' | uniq | grep -f partners.txt + // + // where partners.txt contains + // + // amazon + // aol + // bing + // eBay + // google + // mailru + // mercadolibre + // seznam + // twitter + // yahoo + // yandex + // + // Please update this list as the set of partners changes. + // + PROVIDERS: [ + "amazon-co-uk", + "amazon-de", + "amazon-en-GB", + "amazon-france", + "amazon-it", + "amazon-jp", + "amazondotcn", + "amazondotcom", + "amazondotcom-de", + + "aol-en-GB", + "aol-web-search", + "bing", + + "eBay", + "eBay-de", + "eBay-en-GB", + "eBay-es", + "eBay-fi", + "eBay-france", + "eBay-hu", + "eBay-in", + "eBay-it", + "google", + "google-jp", + "google-ku", + "google-maps-zh-TW", + + "mailru", + + "mercadolibre-ar", + "mercadolibre-cl", + "mercadolibre-mx", + + "seznam-cz", + + "twitter", + "twitter-de", + "twitter-ja", + "yahoo", + "yahoo-NO", + "yahoo-answer-zh-TW", + "yahoo-ar", + "yahoo-bid-zh-TW", + "yahoo-br", + "yahoo-ch", + "yahoo-cl", + "yahoo-de", + "yahoo-en-GB", + "yahoo-es", + "yahoo-fi", + "yahoo-france", + "yahoo-fy-NL", + "yahoo-id", + "yahoo-in", + "yahoo-it", + "yahoo-jp", + "yahoo-jp-auctions", + "yahoo-mx", + "yahoo-sv-SE", + "yahoo-zh-TW", + + "yandex", + "yandex-ru", + "yandex-slovari", + "yandex-tr", + "yandex.by", + "yandex.ru-be", ], SOURCES: [ "abouthome", "contextmenu", "searchbar", "urlbar", ], @@ -1130,41 +1318,41 @@ SearchCountMeasurement.prototype = Objec this.SearchesProvider = function () { Metrics.Provider.call(this); }; this.SearchesProvider.prototype = Object.freeze({ __proto__: Metrics.Provider.prototype, name: "org.mozilla.searches", - measurementTypes: [SearchCountMeasurement], + measurementTypes: [ + SearchCountMeasurement1, + SearchCountMeasurement2, + ], /** * Record that a search occurred. * * @param engine * (string) The search engine used. If the search engine is unknown, * the search will be attributed to "other". * @param source * (string) Where the search was initiated from. Must be one of the - * SearchCountMeasurement.SOURCES values. + * SearchCountMeasurement2.SOURCES values. * * @return Promise<> * The promise is resolved when the storage operation completes. */ recordSearch: function (engine, source) { - let m = this.getMeasurement("counts", 1); + let m = this.getMeasurement("counts", 2); if (m.SOURCES.indexOf(source) == -1) { throw new Error("Unknown source for search: " + source); } - let normalizedEngine = engine.toLowerCase(); - if (m.PARTNER_ENGINES.indexOf(normalizedEngine) == -1) { - normalizedEngine = "other"; - } - + let id = m.interestingEngines[engine] || "other"; + let field = id + "." + source; return this.enqueueStorageOperation(function recordSearch() { - return m.incrementDailyCounter(normalizedEngine + "." + source); + return m.incrementDailyCounter(field); }); }, });
--- a/services/healthreport/tests/xpcshell/test_provider_searches.js +++ b/services/healthreport/tests/xpcshell/test_provider_searches.js @@ -1,75 +1,123 @@ /* Any copyright is dedicated to the Public Domain. * http://creativecommons.org/publicdomain/zero/1.0/ */ "use strict"; const {utils: Cu} = Components; Cu.import("resource://gre/modules/Metrics.jsm"); -Cu.import("resource://gre/modules/services/healthreport/providers.jsm"); +let bsp = Cu.import("resource://gre/modules/services/healthreport/providers.jsm"); + +const DEFAULT_ENGINES = [ + {name: "Amazon.com", identifier: "amazondotcom"}, + {name: "Bing", identifier: "bing"}, + {name: "Google", identifier: "google"}, + {name: "Yahoo", identifier: "yahoo"}, + {name: "Foobar Search", identifier: "foobar"}, +]; +function MockSearchCountMeasurement() { + bsp.SearchCountMeasurement2.call(this); +} +MockSearchCountMeasurement.prototype = { + __proto__: bsp.SearchCountMeasurement2.prototype, + getDefaultEngines: function () { + return DEFAULT_ENGINES; + }, +}; + +function MockSearchesProvider() { + SearchesProvider.call(this); +} +MockSearchesProvider.prototype = { + __proto__: SearchesProvider.prototype, + measurementTypes: [MockSearchCountMeasurement], +}; function run_test() { run_next_test(); } add_test(function test_constructor() { let provider = new SearchesProvider(); run_next_test(); }); add_task(function test_record() { let storage = yield Metrics.Storage("record"); - let provider = new SearchesProvider(); + let provider = new MockSearchesProvider(); yield provider.init(storage); - const ENGINES = [ - "amazon.com", - "bing", - "google", - "yahoo", - "foobar", - ]; - let now = new Date(); - for (let engine of ENGINES) { - yield provider.recordSearch(engine, "abouthome"); - yield provider.recordSearch(engine, "contextmenu"); - yield provider.recordSearch(engine, "searchbar"); - yield provider.recordSearch(engine, "urlbar"); + for (let engine of DEFAULT_ENGINES) { + yield provider.recordSearch(engine.name, "abouthome"); + yield provider.recordSearch(engine.name, "contextmenu"); + yield provider.recordSearch(engine.name, "searchbar"); + yield provider.recordSearch(engine.name, "urlbar"); } // Invalid sources should throw. let errored = false; try { - yield provider.recordSearch("google", "bad source"); + yield provider.recordSearch(DEFAULT_ENGINES[0].name, "bad source"); } catch (ex) { errored = true; } finally { do_check_true(errored); } - let m = provider.getMeasurement("counts", 1); + let m = provider.getMeasurement("counts", 2); let data = yield m.getValues(); do_check_eq(data.days.size, 1); do_check_true(data.days.hasDay(now)); let day = data.days.getDay(now); - for (let engine of ENGINES) { - if (engine == "foobar") { - engine = "other"; + for (let engine of DEFAULT_ENGINES) { + let identifier = engine.identifier; + if (identifier == "foobar") { + identifier = "other"; } for (let source of ["abouthome", "contextmenu", "searchbar", "urlbar"]) { - let field = engine + "." + source; + let field = identifier + "." + source; do_check_true(day.has(field)); do_check_eq(day.get(field), 1); } } yield storage.close(); }); +add_task(function test_includes_other_fields() { + let storage = yield Metrics.Storage("includes_other_fields"); + let provider = new MockSearchesProvider(); + + yield provider.init(storage); + let m = provider.getMeasurement("counts", 2); + + // Register a search against a provider that isn't live in this session. + let id = yield m.storage.registerField(m.id, "test.searchbar", + Metrics.Storage.FIELD_DAILY_COUNTER); + + let testField = "test.searchbar"; + let now = new Date(); + yield m.storage.incrementDailyCounterFromFieldID(id, now); + + // Make sure we don't know about it. + do_check_false(testField in m.fields); + + // But we want to include it in payloads. + do_check_true(m.shouldIncludeField(testField)); + + // And we do so. + let data = yield provider.storage.getMeasurementValues(m.id); + let serializer = m.serializer(m.SERIALIZE_JSON); + let formatted = serializer.daily(data.days.getDay(now)); + do_check_true(testField in formatted); + do_check_eq(formatted[testField], 1); + + yield storage.close(); +});
--- a/services/metrics/dataprovider.jsm +++ b/services/metrics/dataprovider.jsm @@ -177,22 +177,38 @@ Measurement.prototype = Object.freeze({ if (!entry) { throw new Error("Unknown field: " + name); } return entry[1]; }, _configureStorage: function () { - return Task.spawn(function configureFields() { - for (let [name, info] in Iterator(this.fields)) { - this._log.debug("Registering field: " + name + " " + info.type); + let missing = []; + for (let [name, info] in Iterator(this.fields)) { + if (this.storage.hasFieldFromMeasurement(this.id, name)) { + this._fields[name] = + [this.storage.fieldIDFromMeasurement(this.id, name), info.type]; + continue; + } + + missing.push([name, info.type]); + } - let id = yield this.storage.registerField(this.id, name, info.type); - this._fields[name] = [id, info.type]; + if (!missing.length) { + return CommonUtils.laterTickResolvingPromise(); + } + + // We only perform a transaction if we have work to do (to avoid + // extra SQLite overhead). + return this.storage.enqueueTransaction(function registerFields() { + for (let [name, type] of missing) { + this._log.debug("Registering field: " + name + " " + type); + let id = yield this.storage.registerField(this.id, name, type); + this._fields[name] = [id, type]; } }.bind(this)); }, //--------------------------------------------------------------------------- // Data Recording Functions // // Functions in this section are used to record new values against this @@ -326,22 +342,40 @@ Measurement.prototype = Object.freeze({ deleteLastNumeric: function (field) { return this.storage.deleteLastNumericFromFieldID(this.fieldID(field)); }, deleteLastText: function (field) { return this.storage.deleteLastTextFromFieldID(this.fieldID(field)); }, + /** + * This method is used by the default serializers to control whether a field + * is included in the output. + * + * There could be legacy fields in storage we no longer care about. + * + * This method is a hook to allow measurements to change this behavior, e.g., + * to implement a dynamic fieldset. + * + * You will also need to override `fieldType`. + * + * @return (boolean) true if the specified field should be included in + * payload output. + */ + shouldIncludeField: function (field) { + return field in this._fields; + }, + _serializeJSONSingular: function (data) { let result = {"_v": this.version}; for (let [field, data] of data) { // There could be legacy fields in storage we no longer care about. - if (!(field in this._fields)) { + if (!this.shouldIncludeField(field)) { continue; } let type = this.fieldType(field); switch (type) { case this.storage.FIELD_LAST_NUMERIC: case this.storage.FIELD_LAST_TEXT: @@ -362,17 +396,17 @@ Measurement.prototype = Object.freeze({ return result; }, _serializeJSONDay: function (data) { let result = {"_v": this.version}; for (let [field, data] of data) { - if (!(field in this._fields)) { + if (!this.shouldIncludeField(field)) { continue; } let type = this.fieldType(field); switch (type) { case this.storage.FIELD_DAILY_COUNTER: case this.storage.FIELD_DAILY_DISCRETE_NUMERIC:
--- a/services/metrics/providermanager.jsm +++ b/services/metrics/providermanager.jsm @@ -77,25 +77,29 @@ this.ProviderManager.prototype = Object. * value is the resource:// URI to import which makes this type available. * * Example entry: * * FooProvider resource://gre/modules/foo.jsm * * One can register entries in the application's .manifest file. e.g. * - * category healthreport-js-provider FooProvider resource://gre/modules/foo.jsm + * category healthreport-js-provider-default FooProvider resource://gre/modules/foo.jsm + * category healthreport-js-provider-nightly EyeballProvider resource://gre/modules/eyeball.jsm * * Then to load them: * * let reporter = getHealthReporter("healthreport."); - * reporter.registerProvidersFromCategoryManager("healthreport-js-provider"); + * reporter.registerProvidersFromCategoryManager("healthreport-js-provider-default"); + * + * If the category has no defined members, this call has no effect, and no error is raised. * * @param category - * (string) Name of category to query and load from. + * (string) Name of category from which to query and load. + * @return a newly spawned Task. */ registerProvidersFromCategoryManager: function (category) { this._log.info("Registering providers from category: " + category); let cm = Cc["@mozilla.org/categorymanager;1"] .getService(Ci.nsICategoryManager); let promises = []; let enumerator = cm.enumerateCategory(category);
--- a/services/metrics/storage.jsm +++ b/services/metrics/storage.jsm @@ -726,16 +726,25 @@ function MetricsStorageSqliteBackend(con // Maps measurement ID to sets of field IDs. this._fieldsByMeasurement = new Map(); this._queuedOperations = []; this._queuedInProgress = false; } MetricsStorageSqliteBackend.prototype = Object.freeze({ + // Max size (in kibibytes) the WAL log is allowed to grow to before it is + // checkpointed. + // + // This was first deployed in bug 848136. We want a value large enough + // that we aren't checkpointing all the time. However, we want it + // small enough so we don't have to read so much when we open the + // database. + MAX_WAL_SIZE_KB: 512, + FIELD_DAILY_COUNTER: "daily-counter", FIELD_DAILY_DISCRETE_NUMERIC: "daily-discrete-numeric", FIELD_DAILY_DISCRETE_TEXT: "daily-discrete-text", FIELD_DAILY_LAST_NUMERIC: "daily-last-numeric", FIELD_DAILY_LAST_TEXT: "daily-last-text", FIELD_LAST_NUMERIC: "last-numeric", FIELD_LAST_TEXT: "last-text", @@ -1100,16 +1109,53 @@ MetricsStorageSqliteBackend.prototype = * This performs 2 major roles: * * 1) Set up database schema (creates tables). * 2) Synchronize database with local instance. */ _init: function() { let self = this; return Task.spawn(function initTask() { + // 0. Database file and connection configuration. + + // This should never fail. But, we assume the default of 1024 in case it + // does. + let rows = yield self._connection.execute("PRAGMA page_size"); + let pageSize = 1024; + if (rows.length) { + pageSize = rows[0].getResultByIndex(0); + } + + self._log.debug("Page size is " + pageSize); + + // Ensure temp tables are stored in memory, not on disk. + yield self._connection.execute("PRAGMA temp_store=MEMORY"); + + let journalMode; + rows = yield self._connection.execute("PRAGMA journal_mode=WAL"); + if (rows.length) { + journalMode = rows[0].getResultByIndex(0); + } + + self._log.info("Journal mode is " + journalMode); + + if (journalMode == "wal") { + yield self._connection.execute("PRAGMA wal_autocheckpoint=" + + Math.ceil(self.MAX_WAL_SIZE_KB * 1024 / pageSize)); + } else { + if (journalMode != "truncate") { + // Fall back to truncate (which is faster than delete). + yield self._connection.execute("PRAGMA journal_mode=TRUNCATE"); + } + + // And always use full synchronous mode to reduce possibility for data + // loss. + yield self._connection.execute("PRAGMA synchronous=FULL"); + } + // 1. Create the schema. yield self._connection.executeTransaction(function ensureSchema(conn) { let schema = conn.schemaVersion; if (schema == 0) { self._log.info("Creating database schema."); for (let k of self._SCHEMA_STATEMENTS) { @@ -1129,29 +1175,39 @@ MetricsStorageSqliteBackend.prototype = let id = row.getResultByName("id"); let name = row.getResultByName("name"); self._typesByID.set(id, name); self._typesByName.set(name, id); }); // 3. Populate built-in types with database. + let missingTypes = []; for (let type of self._BUILTIN_TYPES) { type = self[type]; if (self._typesByName.has(type)) { continue; } - let params = {name: type}; - yield self._connection.executeCached(SQL.addType, params); - let rows = yield self._connection.executeCached(SQL.getTypeID, params); - let id = rows[0].getResultByIndex(0); + missingTypes.push(type); + } - self._typesByID.set(id, type); - self._typesByName.set(type, id); + // Don't perform DB transaction unless there is work to do. + if (missingTypes.length) { + yield self._connection.executeTransaction(function populateBuiltinTypes() { + for (let type of missingTypes) { + let params = {name: type}; + yield self._connection.executeCached(SQL.addType, params); + let rows = yield self._connection.executeCached(SQL.getTypeID, params); + let id = rows[0].getResultByIndex(0); + + self._typesByID.set(id, type); + self._typesByName.set(type, id); + } + }); } // 4. Obtain measurement info. yield self._connection.execute(SQL.getMeasurements, null, function onRow(row) { let providerID = row.getResultByName("provider_id"); let providerName = row.getResultByName("provider_name"); let measurementID = row.getResultByName("measurement_id"); let measurementName = row.getResultByName("measurement_name"); @@ -1219,16 +1275,29 @@ MetricsStorageSqliteBackend.prototype = let self = this; return this.enqueueOperation(function doCompact() { self._connection.discardCachedStatements(); return self._connection.shrinkMemory(); }); }, /** + * Checkpoint writes requiring flush to disk. + * + * This is called to persist queued and non-flushed writes to disk. + * It will force an fsync, so it is expensive and should be used + * sparingly. + */ + checkpoint: function () { + return this.enqueueOperation(function checkpoint() { + return this._connection.execute("PRAGMA wal_checkpoint"); + }.bind(this)); + }, + + /** * Ensure a field ID matches a specified type. * * This is called internally as part of adding values to ensure that * the type of a field matches the operation being performed. */ _ensureFieldType: function (id, type) { let info = this._fieldsByID.get(id);
--- a/services/metrics/tests/xpcshell/test_metrics_provider.js +++ b/services/metrics/tests/xpcshell/test_metrics_provider.js @@ -268,10 +268,26 @@ add_task(function test_serialize_json_de do_check_eq(formatted["daily-last-numeric"], 4); do_check_eq(formatted["daily-last-text"], "apple"); formatted = serializer.daily(data.days.getDay(yesterday)); do_check_eq(formatted["daily-last-numeric"], 5); do_check_eq(formatted["daily-last-text"], "orange"); + // Now let's turn off a field so that it's present in the DB + // but not present in the output. + let called = false; + let excluded = "daily-last-numeric"; + Object.defineProperty(m, "shouldIncludeField", { + value: function fakeShouldIncludeField(field) { + called = true; + return field != excluded; + }, + }); + + let limited = serializer.daily(data.days.getDay(yesterday)); + do_check_true(called); + do_check_false(excluded in limited); + do_check_eq(formatted["daily-last-text"], "orange"); + yield provider.storage.close(); });
--- a/toolkit/components/search/nsSearchService.js +++ b/toolkit/components/search/nsSearchService.js @@ -1069,16 +1069,19 @@ function Engine(aLocation, aSourceDataTy ERROR("Engine location is neither a File nor a URI object", Cr.NS_ERROR_INVALID_ARG); } Engine.prototype = { // The engine's alias (can be null). Initialized to |undefined| to indicate // not-initialized-from-engineMetadataService. _alias: undefined, + // A distribution-unique identifier for the engine. Either null or set + // when loaded. See getter. + _identifier: undefined, // The data describing the engine. Is either an array of bytes, for Sherlock // files, or an XML document element, for XML plugins. _data: null, // The engine's data type. See data types (DATA_) defined above. _dataType: null, // Whether or not the engine is readonly. _readOnly: true, // The engine's description @@ -2263,16 +2266,48 @@ Engine.prototype = { return this._alias; }, set alias(val) { this._alias = val; engineMetadataService.setAttr(this, "alias", val); notifyAction(this, SEARCH_ENGINE_CHANGED); }, + /** + * Return the built-in identifier of app-provided engines. + * + * Note that this identifier is substantially similar to _id, with the + * following exceptions: + * + * * There is no trailing file extension. + * * There is no [app] prefix. + * + * @return a string identifier, or null. + */ + get identifier() { + if (this._identifier !== undefined) { + return this._identifier; + } + + // No identifier if If the engine isn't app-provided + if (!this._isInAppDir && !this._isInJAR) { + return this._identifier = null; + } + + let leaf = this._getLeafName(); + ENSURE_WARN(leaf, "identifier: app-provided engine has no leafName"); + + // Strip file extension. + let ext = leaf.lastIndexOf("."); + if (ext == -1) { + return this._identifier = leaf; + } + return this._identifier = leaf.substring(0, ext); + }, + get description() { return this._description; }, get hidden() { if (this._hidden === null) this._hidden = engineMetadataService.getAttr(this, "hidden") || false; return this._hidden; @@ -2306,55 +2341,69 @@ Engine.prototype = { return this._file.path; if (this._uri) return this._uri.spec; return ""; }, + /** + * @return the leaf name of the filename or URI of this plugin, + * or null if no file or URI is known. + */ + _getLeafName: function () { + if (this._file) { + return this._file.leafName; + } + if (this._uri && this._uri instanceof Ci.nsIURL) { + return this._uri.fileName; + } + return null; + }, + // The file that the plugin is loaded from is a unique identifier for it. We // use this as the identifier to store data in the sqlite database __id: null, get _id() { - if (this.__id) + if (this.__id) { return this.__id; + } + + let leafName = this._getLeafName(); // Treat engines loaded from JARs the same way we treat app shipped // engines. // Theoretically, these could also come from extensions, but there's no // real way for extensions to register their chrome locations at the // moment, so let's not deal with that case. // This means we're vulnerable to conflicts if a file loaded from a JAR // has the same filename as a file loaded from the app dir, but with a // different engine name. People using the JAR functionality should be // careful not to do that! if (this._isInAppDir || this._isInJAR) { - let leafName; - if (this._file) - leafName = this._file.leafName; - else { - // If we've reached this point, we must be loaded from a JAR, which - // also means we should have a URL. - ENSURE_WARN(this._isInJAR && (this._uri instanceof Ci.nsIURL), - "_id: not inJAR, or no URI", Cr.NS_ERROR_UNEXPECTED); - leafName = this._uri.fileName; - } - + // App dir and JAR engines should always have leafNames + ENSURE_WARN(leafName, "_id: no leafName for appDir or JAR engine", + Cr.NS_ERROR_UNEXPECTED); return this.__id = "[app]/" + leafName; } - ENSURE_WARN(this._file, "_id: no _file!", Cr.NS_ERROR_UNEXPECTED); - - if (this._isInProfile) - return this.__id = "[profile]/" + this._file.leafName; + if (this._isInProfile) { + ENSURE_WARN(leafName, "_id: no leafName for profile engine", + Cr.NS_ERROR_UNEXPECTED); + return this.__id = "[profile]/" + leafName; + } + + // If the engine isn't a JAR engine, it should have a file. + ENSURE_WARN(this._file, "_id: no _file for non-JAR engine", + Cr.NS_ERROR_UNEXPECTED); // We're not in the profile or appdir, so this must be an extension-shipped // plugin. Use the full filename. - return this.__id = this._file.path; + return this.__id = this._file.path; }, get _installLocation() { if (this.__installLocation === null) { if (!this._file) { ENSURE_WARN(this._uri, "Engines without files must have URIs", Cr.NS_ERROR_UNEXPECTED); this.__installLocation = SEARCH_JAR;
new file mode 100644 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_identifiers.js @@ -0,0 +1,60 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test that a search engine's identifier can be extracted from the filename. + */ + +"use strict"; + +const Ci = Components.interfaces; +const SEARCH_APP_DIR = 1; + +function run_test() { + removeMetadata(); + removeCacheFile(); + do_load_manifest("data/chrome.manifest"); + + let url = "chrome://testsearchplugin/locale/searchplugins/"; + Services.prefs.setCharPref("browser.search.jarURIs", url); + Services.prefs.setBoolPref("browser.search.loadFromJars", true); + + updateAppInfo(); + + run_next_test(); +} + +add_test(function test_identifier() { + let engineFile = gProfD.clone(); + engineFile.append("searchplugins"); + engineFile.append("test-search-engine.xml"); + engineFile.parent.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY); + + // Copy the test engine to the test profile. + let engineTemplateFile = do_get_file("data/engine.xml"); + engineTemplateFile.copyTo(engineFile.parent, "test-search-engine.xml"); + + let search = Services.search.init(function initComplete(aResult) { + do_print("init'd search service"); + do_check_true(Components.isSuccessCode(aResult)); + + let profileEngine = Services.search.getEngineByName("Test search engine"); + let jarEngine = Services.search.getEngineByName("bug645970"); + + do_check_true(profileEngine instanceof Ci.nsISearchEngine); + do_check_true(jarEngine instanceof Ci.nsISearchEngine); + + // An engine loaded from the profile directory won't have an identifier, + // because it's not built-in. + do_check_eq(profileEngine.identifier, null); + + // An engine loaded from a JAR will have an identifier corresponding to + // the filename inside the JAR. (In this case it's the same as the name.) + do_check_eq(jarEngine.identifier, "bug645970"); + + removeMetadata(); + removeCacheFile(); + run_next_test(); + }); +}); +
--- a/toolkit/components/search/tests/xpcshell/xpcshell.ini +++ b/toolkit/components/search/tests/xpcshell/xpcshell.ini @@ -2,16 +2,17 @@ head = head_search.js tail = firefox-appdir = browser [test_nocache.js] [test_645970.js] # Bug 845190: Too many intermittent assertions on Linux (ASSERTION: thread pool wasn't shutdown) skip-if = debug && os == "linux" +[test_identifiers.js] [test_init_async_multiple.js] [test_init_async_multiple_then_sync.js] [test_json_cache.js] [test_migratedb.js] [test_nodb.js] [test_nodb_pluschanges.js] [test_purpose.js]
--- a/toolkit/components/telemetry/Histograms.json +++ b/toolkit/components/telemetry/Histograms.json @@ -3021,16 +3021,22 @@ "description": "Time (ms) it takes FHR to collect daily data." }, "HEALTHREPORT_SHUTDOWN_MS": { "kind": "exponential", "high": "20000", "n_buckets": 15, "description": "Time (ms) it takes FHR to shut down." }, + "HEALTHREPORT_POST_COLLECT_CHECKPOINT_MS": { + "kind": "exponential", + "high": "20000", + "n_buckets": 15, + "description": "Time (ms) for a WAL checkpoint after collecting all measurements." + }, "POPUP_NOTIFICATION_MAINACTION_TRIGGERED_MS": { "kind": "linear", "low": 25, "high": "80 * 25", "n_buckets": "80 + 1", "description": "The time (in milliseconds) after showing a PopupNotification that the mainAction was first triggered" } }